From 71c8d3fd4100ba59f6d0eb7f0377eb83bc26dd4a Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 8 Feb 2012 15:13:08 -0800 Subject: [PATCH 001/374] reorganize directory structure From 76cacbd9918397c85c7fce56571a8658dd9c827c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 14 Feb 2012 15:51:32 -0800 Subject: [PATCH 002/374] Added coherence tile function defs, with traits and constants --- uncore/coherence.scala | 128 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 uncore/coherence.scala diff --git a/uncore/coherence.scala b/uncore/coherence.scala new file mode 100644 index 00000000..460214e1 --- /dev/null +++ b/uncore/coherence.scala @@ -0,0 +1,128 @@ +package Top { + +import Chisel._ +import Constants._ + +class TransactionInit extends Bundle { + val ttype = Bits(width = 2) + val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) + val address = Bits(width = PADDR_BITS) + val data = Bits(width = MEM_DATA_BITS) +} + +class TransactionAbort extends Bundle { + val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) +} + +class ProbeRequest extends Bundle { + val ptype = Bits(width = 2) + val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) + val address = Bits(width = PADDR_BITS) +} + +class ProbeReply extends Bundle { + val ptype = Bits(width = 2) + val hasData = Bool() + val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) + val data = Bits(width = MEM_DATA_BITS) +} + +class TransactionReply extends Bundle { + val ttype = Bits(width = 2) + val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) + val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) + val data = Bits(width = MEM_DATA_BITS) +} + +class TransactionFinish extends Bundle { + val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) +} + +class ioTileLink extends Bundle { + val xact_init = new TransactionInit().asOutput + val xact_abort = new TransactionAbort().asInput + val probe_req = new ProbeRequest().asInput + val probe_rep = new ProbeReply().asOutput + val xact_rep = new TransactionReply().asInput + val xact_finish = new TransactionFinish().asOutput +} + +trait ThreeStateIncoherence { + val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } + + def cpuCmdToRW( cmd: Bits): (Bool, Bool) = { + val store = (cmd === M_XWR) + val load = (cmd === M_XRD) + val amo = cmd(3).toBool + val read = load || amo || (cmd === M_PFR) + val write = store || amo || (cmd === M_PFW) + (read, write) + } + + def isHit ( cmd: Bits, state: UFix): Bool = { + val (read, write) = cpuCmdToRW(cmd) + ( state === tileClean || state === tileDirty) + } + + def isValid (state: UFix): Bool = { + state != tileInvalid + } + + def needsWriteback (state: UFix): Bool = { + state === tileDirty + } + + def newStateOnWriteback() = tileInvalid + def newStateOnFlush() = tileInvalid + def newState(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, tileDirty, Mux(read, Mux(state === tileDirty, tileDirty, tileClean), tileInvalid)) + } + def newStateOnHit(cmd: Bits, state: UFix): UFix = newState(cmd, state) + def newStateOnPrimaryMiss(cmd: Bits): UFix = newState(cmd, tileInvalid) + def newStateOnSecondaryMiss(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, tileDirty, state) + } + +} + +trait FourStateCoherence { + + val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(4){ UFix() } + val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(3){ UFix() } + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } + + def isHit ( cmd: Bits, state: UFix): Bool = { + val is_hit = Bool(false) + switch(cmd) { + is(M_XRD) { + is_hit := state === tileShared || + state === tileExclusiveClean || + state === tileExclusiveDirty + } + is(M_XWR) { + is_hit := state === tileExclusiveClean || + state === tileExclusiveDirty + } + } + is_hit + } + + def needsWriteback (state: UFix): Bool = { + state === tileExclusiveDirty + } + + def needsSecondaryXact (cmd: Bits, outstanding: TransactionInit): Bool + + def getMetaUpdateOnProbe (incoming: ProbeRequest): Bits = { + val state = UFix(0) + switch(incoming.ptype) { + is(probeInvalidate) { state := tileInvalid } + is(probeDowngrade) { state := tileShared } + } + state.toBits + } +} + +} From ecbf47fe16327ae8a6c4453556a42b989df59dea Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 15 Feb 2012 13:54:36 -0800 Subject: [PATCH 003/374] Replace nbcache manipulation of meta state bits with abstracted functions --- uncore/coherence.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 460214e1..7419555e 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -76,7 +76,7 @@ trait ThreeStateIncoherence { def newStateOnFlush() = tileInvalid def newState(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, tileDirty, Mux(read, Mux(state === tileDirty, tileDirty, tileClean), tileInvalid)) + Mux(write, tileDirty, Mux(read, Mux(state === tileDirty, tileDirty, tileClean), state)) } def newStateOnHit(cmd: Bits, state: UFix): UFix = newState(cmd, state) def newStateOnPrimaryMiss(cmd: Bits): UFix = newState(cmd, tileInvalid) From cfb818aa97e0c96a0b8bb033a5680d442b344433 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 16 Feb 2012 12:59:38 -0800 Subject: [PATCH 004/374] Abstract class for coherence policies --- uncore/coherence.scala | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 7419555e..b55ac9a1 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -47,9 +47,7 @@ class ioTileLink extends Bundle { val xact_finish = new TransactionFinish().asOutput } -trait ThreeStateIncoherence { - val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } - +trait CoherencePolicy { def cpuCmdToRW( cmd: Bits): (Bool, Bool) = { val store = (cmd === M_XWR) val load = (cmd === M_XRD) @@ -58,6 +56,10 @@ trait ThreeStateIncoherence { val write = store || amo || (cmd === M_PFW) (read, write) } +} + +trait ThreeStateIncoherence extends CoherencePolicy { + val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } def isHit ( cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) @@ -87,32 +89,35 @@ trait ThreeStateIncoherence { } -trait FourStateCoherence { +trait FourStateCoherence extends CoherencePolicy { val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(4){ UFix() } val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(3){ UFix() } val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } def isHit ( cmd: Bits, state: UFix): Bool = { - val is_hit = Bool(false) - switch(cmd) { - is(M_XRD) { - is_hit := state === tileShared || - state === tileExclusiveClean || - state === tileExclusiveDirty - } - is(M_XWR) { - is_hit := state === tileExclusiveClean || - state === tileExclusiveDirty - } - } - is_hit + val (read, write) = cpuCmdToRW(cmd) + ((read && ( state === tileShared || state === tileExclusiveClean || state === tileExclusiveDirty)) || + (write && (state === tileExclusiveClean || state === tileExclusiveDirty))) + } + + def isValid (state: UFix): Bool = { + state != tileInvalid } def needsWriteback (state: UFix): Bool = { state === tileExclusiveDirty } + def newStateOnWriteback() = tileInvalid + def newStateOnFlush() = tileInvalid + + // TODO: New funcs as compared to incoherent protocol: + def newState(cmd: Bits, state: UFix): UFix + def newStateOnHit(cmd: Bits, state: UFix): UFix + def newStateOnPrimaryMiss(cmd: Bits): UFix + def newStateOnSecondaryMiss(cmd: Bits, state: UFix): UFix + def needsSecondaryXact (cmd: Bits, outstanding: TransactionInit): Bool def getMetaUpdateOnProbe (incoming: ProbeRequest): Bits = { From b9c42a80c87dfae75110004a651d3bccc3217bc9 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 22 Feb 2012 10:12:13 -0800 Subject: [PATCH 005/374] Added coherence message type enums --- uncore/coherence.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index b55ac9a1..9fcf32eb 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -4,7 +4,7 @@ import Chisel._ import Constants._ class TransactionInit extends Bundle { - val ttype = Bits(width = 2) + val ttype = Bits(width = TTYPE_BITS) val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) val address = Bits(width = PADDR_BITS) val data = Bits(width = MEM_DATA_BITS) @@ -15,20 +15,20 @@ class TransactionAbort extends Bundle { } class ProbeRequest extends Bundle { - val ptype = Bits(width = 2) + val ptype = Bits(width = PTYPE_BITS) val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) val address = Bits(width = PADDR_BITS) } class ProbeReply extends Bundle { - val ptype = Bits(width = 2) + val ptype = Bits(width = PTYPE_BITS) val hasData = Bool() val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) val data = Bits(width = MEM_DATA_BITS) } class TransactionReply extends Bundle { - val ttype = Bits(width = 2) + val ttype = Bits(width = TTYPE_BITS) val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) val data = Bits(width = MEM_DATA_BITS) From 33a26424bd1cdc52298a97dcc30952b8fe30d05a Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 22 Feb 2012 12:14:57 -0800 Subject: [PATCH 006/374] Refining tilelink interface --- uncore/coherence.scala | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 9fcf32eb..cfde62ba 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -39,12 +39,12 @@ class TransactionFinish extends Bundle { } class ioTileLink extends Bundle { - val xact_init = new TransactionInit().asOutput - val xact_abort = new TransactionAbort().asInput - val probe_req = new ProbeRequest().asInput - val probe_rep = new ProbeReply().asOutput - val xact_rep = new TransactionReply().asInput - val xact_finish = new TransactionFinish().asOutput + val xact_init = (new ioDecoupled) { new TransactionInit() }.flip + val xact_abort = (new ioDecoupled) { new TransactionAbort() } + val probe_req = (new ioDecoupled) { new ProbeRequest() } + val probe_rep = (new ioDecoupled) { new ProbeReply() }.flip + val xact_rep = (new ioDecoupled) { new TransactionReply() } + val xact_finish = (new ioDecoupled) { new TransactionFinish() }.flip } trait CoherencePolicy { @@ -130,4 +130,6 @@ trait FourStateCoherence extends CoherencePolicy { } } + + } From c3646e2e6411c3003a9683b4674a023b780c3848 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 22 Feb 2012 18:24:52 -0800 Subject: [PATCH 007/374] Improved TileIO organization, beginnings of hub implementation --- uncore/coherence.scala | 92 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 86 insertions(+), 6 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index cfde62ba..776a64e0 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -24,6 +24,9 @@ class ProbeReply extends Bundle { val ptype = Bits(width = PTYPE_BITS) val hasData = Bool() val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) +} + +class ProbeReplyData extends Bundle { val data = Bits(width = MEM_DATA_BITS) } @@ -31,6 +34,9 @@ class TransactionReply extends Bundle { val ttype = Bits(width = TTYPE_BITS) val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) +} + +class TransactionReplyData extends Bundle { val data = Bits(width = MEM_DATA_BITS) } @@ -39,12 +45,14 @@ class TransactionFinish extends Bundle { } class ioTileLink extends Bundle { - val xact_init = (new ioDecoupled) { new TransactionInit() }.flip - val xact_abort = (new ioDecoupled) { new TransactionAbort() } - val probe_req = (new ioDecoupled) { new ProbeRequest() } - val probe_rep = (new ioDecoupled) { new ProbeReply() }.flip - val xact_rep = (new ioDecoupled) { new TransactionReply() } - val xact_finish = (new ioDecoupled) { new TransactionFinish() }.flip + val xact_init = (new ioDecoupled) { new TransactionInit() }.flip + val xact_abort = (new ioDecoupled) { new TransactionAbort() } + val probe_req = (new ioDecoupled) { new ProbeRequest() } + val probe_rep = (new ioDecoupled) { new ProbeReply() }.flip + val probe_rep_data = (new ioDecoupled) { new ProbeReplyData() }.flip + val xact_rep = (new ioDecoupled) { new TransactionReply() } + val xact_rep_data = (new ioDecoupled) { new TransactionReplyData() } + val xact_finish = (new ioDecoupled) { new TransactionFinish() }.flip } trait CoherencePolicy { @@ -130,6 +138,78 @@ trait FourStateCoherence extends CoherencePolicy { } } +class XactTracker(id: Int) extends Component { + val io = new Bundle { + val xact_init = (new ioDecoupled) { new TransactionInit() } + val probe_rep = (new ioDecoupled) { new ProbeReply() } + val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip + val xact_rep = (new ioDecoupled) { new TransactionReply() }.flip + val mem_req = (new ioDecoupled) { new MemReq() }.flip + val xact_finish = Bool(INPUT) + val tile_id_in = Bits(TILE_ID_BITS, INPUT) + val tile_id_out = Bits(TILE_ID_BITS, OUTPUT) + val ongoing_addr = Bits(PADDR_BITS, OUTPUT) + val busy = Bool(OUTPUT) + } + + val valid = Reg(resetVal = Bool(false)) + val addr = Reg{ Bits() } + val ttype = Reg{ Bits() } + val tile_id = Reg{ Bits() } + val tile_xact_id = Reg{ Bits() } + val probe_done = Reg{ Bits() } + +} + +abstract class CoherenceHub extends Component + +class CoherenceHubNoDir extends CoherenceHub { + val io = new Bundle { + val tiles = Vec(NTILES) { new ioTileLink() } + val mem = new ioDCache().flip + } + + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) + + // In parallel, every cycle: nack conflicting transactions, free finished ones + for( j <- 0 until NTILES ) { + val init = io.tiles(j).xact_init + val abort = io.tiles(j).xact_abort + val conflicts = Bits(width = NGLOBAL_XACTS) + val busys = Bits(width = NGLOBAL_XACTS) + for( i <- 0 until NGLOBAL_XACTS) { + val t = trackerList(i).io + busys(i) := t.busy + conflicts(i) := t.busy && init.valid && (t.ongoing_addr === init.bits.address) + } + abort.valid := conflicts.orR || busys.andR + abort.bits.tileTransactionID := init.bits.tileTransactionID + //if abort.rdy, init.pop() + + } + for( i <- 0 until NGLOBAL_XACTS) { + val t = trackerList(i).io + val freed = Bits(width = NTILES) + for( j <- 0 until NTILES ) { + val finish = io.tiles(j).xact_finish + freed(j) := finish.valid && (UFix(i) === finish.bits.globalTransactionID) + } + t.xact_finish := freed.orR + //finish.pop() + } + + // Forward memory responses from mem to tile + //for( j <- until NTILES ) { + // tiles(j).xact_rep.ttype = + // tiles(j).xact_rep.tileTransactionID = + // tiles(j).xact_rep.globalTransactionID = + // val data = Bits + // + // Pick a single request of these types to process + //val xact_init_arb = (new Arbiter(NTILES)) { new TransactionInit() } + //val probe_reply_arb = (new Arbiter(NTILES)) { new ProbeReply() } } + +} From ffb88467db080f48cb2d62f7a308613b50440cde Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 23 Feb 2012 17:49:28 -0800 Subject: [PATCH 008/374] finished xact_rep transactor in coherence hub --- uncore/coherence.scala | 96 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 78 insertions(+), 18 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 776a64e0..3744fdb6 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -2,6 +2,7 @@ package Top { import Chisel._ import Constants._ +import hwacha.GenArray class TransactionInit extends Bundle { val ttype = Bits(width = TTYPE_BITS) @@ -146,10 +147,12 @@ class XactTracker(id: Int) extends Component { val xact_rep = (new ioDecoupled) { new TransactionReply() }.flip val mem_req = (new ioDecoupled) { new MemReq() }.flip val xact_finish = Bool(INPUT) - val tile_id_in = Bits(TILE_ID_BITS, INPUT) - val tile_id_out = Bits(TILE_ID_BITS, OUTPUT) - val ongoing_addr = Bits(PADDR_BITS, OUTPUT) val busy = Bool(OUTPUT) + val addr = Bits(PADDR_BITS, OUTPUT) + val tile_id = Bits(TILE_ID_BITS, OUTPUT) + val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) + val sharer_count = Bits(TILE_ID_BITS, OUTPUT) + val ttype = Bits(TTYPE_BITS, OUTPUT) } val valid = Reg(resetVal = Bool(false)) @@ -164,47 +167,104 @@ class XactTracker(id: Int) extends Component { abstract class CoherenceHub extends Component class CoherenceHubNoDir extends CoherenceHub { + + def coherenceConflict(addr1: Bits, addr2: Bits): Bool = { + addr1(PADDR_BITS-1, OFFSET_BITS) === addr2(PADDR_BITS-1, OFFSET_BITS) + } + def getTransactionReplyType(ttype: UFix, count: UFix): Bits = { + val ret = Wire() { Bits(width = TTYPE_BITS) } + switch (ttype) { + is(X_READ_SHARED) { ret := Mux(count > UFix(0), X_READ_SHARED, X_READ_EXCLUSIVE) } + is(X_READ_EXCLUSIVE) { ret := X_READ_EXCLUSIVE } + is(X_READ_UNCACHED) { ret := X_READ_UNCACHED } + is(X_WRITE_UNCACHED) { ret := X_WRITE_UNCACHED } + } + ret + } + val io = new Bundle { val tiles = Vec(NTILES) { new ioTileLink() } val mem = new ioDCache().flip } - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) + val busy_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + val addr_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } + val tile_id_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val tile_xact_id_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } + val sh_count_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val ttype_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TTYPE_BITS)} } + val free_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + for( i <- 0 until NGLOBAL_XACTS) { + busy_arr.write( UFix(i), trackerList(i).io.busy) + addr_arr.write( UFix(i), trackerList(i).io.addr) + tile_id_arr.write( UFix(i), trackerList(i).io.tile_id) + tile_xact_id_arr.write(UFix(i), trackerList(i).io.tile_xact_id) + ttype_arr.write( UFix(i), trackerList(i).io.ttype) + sh_count_arr.write( UFix(i), trackerList(i).io.sharer_count) + trackerList(i).io.xact_finish := free_arr.read(UFix(i)) + } // In parallel, every cycle: nack conflicting transactions, free finished ones for( j <- 0 until NTILES ) { val init = io.tiles(j).xact_init val abort = io.tiles(j).xact_abort val conflicts = Bits(width = NGLOBAL_XACTS) - val busys = Bits(width = NGLOBAL_XACTS) for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - busys(i) := t.busy - conflicts(i) := t.busy && init.valid && (t.ongoing_addr === init.bits.address) + conflicts(i) := t.busy(i) && coherenceConflict(t.addr, init.bits.address) } - abort.valid := conflicts.orR || busys.andR + abort.valid := init.valid && (conflicts.orR || busy_arr.flatten().andR) abort.bits.tileTransactionID := init.bits.tileTransactionID - //if abort.rdy, init.pop() - + // TODO: + // Reg(aborted) := (abort.ready && abort.valid) + // Reg(allocated) : = had_priority(j) & !(abort.ready && abort.valid) + // init.rdy = aborted || allocated } + +/* +// Todo: which implementation is clearer? for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io val freed = Bits(width = NTILES) for( j <- 0 until NTILES ) { val finish = io.tiles(j).xact_finish - freed(j) := finish.valid && (UFix(i) === finish.bits.globalTransactionID) + free(j) := finish.valid && (UFix(i) === finish.bits.globalTransactionID) + finish.ready := Bool(true) // finsh.pop() } t.xact_finish := freed.orR - //finish.pop() + } +*/ + + free_arr := Bits(0, width=NGLOBAL_XACTS) + for( j <- 0 until NTILES ) { + val finish = io.tiles(j).xact_finish + when(finish.valid) { + free_arr.write(finish.bits.globalTransactionID, Bool(true)) + } + finish.ready := Bool(true) } // Forward memory responses from mem to tile - //for( j <- until NTILES ) { - // tiles(j).xact_rep.ttype = - // tiles(j).xact_rep.tileTransactionID = - // tiles(j).xact_rep.globalTransactionID = - // val data = Bits - // + val xrep_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) + val xrep_cnt_next = xrep_cnt + UFix(1) + when (io.mem.resp_val) { xrep_cnt := xrep_cnt_next } + val idx = io.mem.resp_tag + val readys = Bits(width = NTILES) + for( j <- 0 until NTILES ) { + io.tiles(j).xact_rep.bits.ttype := getTransactionReplyType(ttype_arr.read(idx), sh_count_arr.read(idx)) + io.tiles(j).xact_rep.bits.tileTransactionID := tile_xact_id_arr.read(idx) + io.tiles(j).xact_rep.bits.globalTransactionID := idx + io.tiles(j).xact_rep_data.bits.data := io.mem.resp_data + readys := Mux(xrep_cnt === UFix(0), io.tiles(j).xact_rep.ready && io.tiles(j).xact_rep_data.ready, io.tiles(j).xact_rep_data.ready) + val this_rep_valid = UFix(j) === tile_id_arr.read(idx) && io.mem.resp_val + io.tiles(j).xact_rep.valid := this_rep_valid && xrep_cnt === UFix(0) + io.tiles(j).xact_rep_data.valid := this_rep_valid + } + // If there were a ready signal due to e.g. intervening network: + //io.mem.resp_rdy := readys(tile_id_arr.read(idx)).xact_rep.ready + + // Pick a single request of these types to process //val xact_init_arb = (new Arbiter(NTILES)) { new TransactionInit() } //val probe_reply_arb = (new Arbiter(NTILES)) { new ProbeReply() } From 1dcf25586f9997f512cf88f025174262b12f9c52 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 23 Feb 2012 18:12:50 -0800 Subject: [PATCH 009/374] finished xact_finish and xact_abort transactors in coherence hub --- uncore/coherence.scala | 32 ++++++++------------------------ 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 3744fdb6..534c4bb9 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -205,7 +205,9 @@ class CoherenceHubNoDir extends CoherenceHub { trackerList(i).io.xact_finish := free_arr.read(UFix(i)) } - // In parallel, every cycle: nack conflicting transactions, free finished ones + // Nack conflicting transaction init attempts + val aborting = Wire() { Bits(width = NTILES) } + val initiating = Wire() { Bits(width = NTILES) } for( j <- 0 until NTILES ) { val init = io.tiles(j).xact_init val abort = io.tiles(j).xact_abort @@ -214,34 +216,16 @@ class CoherenceHubNoDir extends CoherenceHub { val t = trackerList(i).io conflicts(i) := t.busy(i) && coherenceConflict(t.addr, init.bits.address) } - abort.valid := init.valid && (conflicts.orR || busy_arr.flatten().andR) + aborting(j) := (conflicts.orR || busy_arr.flatten().andR) + abort.valid := init.valid && aborting abort.bits.tileTransactionID := init.bits.tileTransactionID - // TODO: - // Reg(aborted) := (abort.ready && abort.valid) - // Reg(allocated) : = had_priority(j) & !(abort.ready && abort.valid) - // init.rdy = aborted || allocated + init.ready := aborting(j) || initiating(j) } - -/* -// Todo: which implementation is clearer? - for( i <- 0 until NGLOBAL_XACTS) { - val t = trackerList(i).io - val freed = Bits(width = NTILES) - for( j <- 0 until NTILES ) { - val finish = io.tiles(j).xact_finish - free(j) := finish.valid && (UFix(i) === finish.bits.globalTransactionID) - finish.ready := Bool(true) // finsh.pop() - } - t.xact_finish := freed.orR - } -*/ - free_arr := Bits(0, width=NGLOBAL_XACTS) + // Free finished transactions for( j <- 0 until NTILES ) { val finish = io.tiles(j).xact_finish - when(finish.valid) { - free_arr.write(finish.bits.globalTransactionID, Bool(true)) - } + free_arr.write(finish.bits.globalTransactionID, finish.valid) finish.ready := Bool(true) } From b3cf8f3f354b064c139ae040038d70c438542d19 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sat, 25 Feb 2012 12:56:09 -0800 Subject: [PATCH 010/374] Better abstraction of data bundles --- uncore/coherence.scala | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 534c4bb9..579af518 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -4,13 +4,27 @@ import Chisel._ import Constants._ import hwacha.GenArray +class HubMemReq extends Bundle { + val rw = Bool() + val addr = UFix(width = PADDR_BITS-OFFSET_BITS) + val tag = Bits(width = GLOBAL_XACT_ID_BITS) + // Figure out which data-in port to pull from + val data_idx = Bits(width = TILE_ID_BITS) + val is_probe_rep = Bool() +} + +class MemData extends Bundle { + val data = Bits(width = MEM_DATA_BITS) +} + class TransactionInit extends Bundle { val ttype = Bits(width = TTYPE_BITS) val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) val address = Bits(width = PADDR_BITS) - val data = Bits(width = MEM_DATA_BITS) } +class TransactionInitData extends MemData + class TransactionAbort extends Bundle { val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) } @@ -27,9 +41,7 @@ class ProbeReply extends Bundle { val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) } -class ProbeReplyData extends Bundle { - val data = Bits(width = MEM_DATA_BITS) -} +class ProbeReplyData extends MemData class TransactionReply extends Bundle { val ttype = Bits(width = TTYPE_BITS) @@ -37,9 +49,7 @@ class TransactionReply extends Bundle { val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) } -class TransactionReplyData extends Bundle { - val data = Bits(width = MEM_DATA_BITS) -} +class TransactionReplyData extends MemData class TransactionFinish extends Bundle { val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) @@ -47,6 +57,7 @@ class TransactionFinish extends Bundle { class ioTileLink extends Bundle { val xact_init = (new ioDecoupled) { new TransactionInit() }.flip + val xact_init_data = (new ioDecoupled) { new TransactionInitData() }.flip val xact_abort = (new ioDecoupled) { new TransactionAbort() } val probe_req = (new ioDecoupled) { new ProbeRequest() } val probe_rep = (new ioDecoupled) { new ProbeReply() }.flip From dfdcb7c696df600ddc9a234bf189fbd2600b3202 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sat, 25 Feb 2012 15:27:09 -0800 Subject: [PATCH 011/374] Better foldR --- uncore/coherence.scala | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 579af518..bb2f7d65 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -18,41 +18,43 @@ class MemData extends Bundle { } class TransactionInit extends Bundle { - val ttype = Bits(width = TTYPE_BITS) - val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) + val t_type = Bits(width = TTYPE_BITS) + val has_data = Bool() + val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) val address = Bits(width = PADDR_BITS) } class TransactionInitData extends MemData class TransactionAbort extends Bundle { - val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) + val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) } class ProbeRequest extends Bundle { - val ptype = Bits(width = PTYPE_BITS) - val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) + val p_type = Bits(width = PTYPE_BITS) + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) val address = Bits(width = PADDR_BITS) } class ProbeReply extends Bundle { - val ptype = Bits(width = PTYPE_BITS) - val hasData = Bool() - val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) + val p_type = Bits(width = PTYPE_BITS) + val has_data = Bool() + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } class ProbeReplyData extends MemData class TransactionReply extends Bundle { - val ttype = Bits(width = TTYPE_BITS) - val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) - val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) + val t_type = Bits(width = TTYPE_BITS) + val has_data = Bool() + val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } class TransactionReplyData extends MemData class TransactionFinish extends Bundle { - val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } class ioTileLink extends Bundle { @@ -142,7 +144,7 @@ trait FourStateCoherence extends CoherencePolicy { def getMetaUpdateOnProbe (incoming: ProbeRequest): Bits = { val state = UFix(0) - switch(incoming.ptype) { + switch(incoming.p_type) { is(probeInvalidate) { state := tileInvalid } is(probeDowngrade) { state := tileShared } } From 8856e2b8bb7b5c5851d227704e8182c50ee1f01a Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sat, 25 Feb 2012 15:27:53 -0800 Subject: [PATCH 012/374] More stylish bundle param names, some hub progress --- uncore/coherence.scala | 105 ++++++++++++++++++++++++++++++----------- 1 file changed, 78 insertions(+), 27 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index bb2f7d65..d7597cdf 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -154,23 +154,27 @@ trait FourStateCoherence extends CoherencePolicy { class XactTracker(id: Int) extends Component { val io = new Bundle { - val xact_init = (new ioDecoupled) { new TransactionInit() } - val probe_rep = (new ioDecoupled) { new ProbeReply() } - val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip - val xact_rep = (new ioDecoupled) { new TransactionReply() }.flip - val mem_req = (new ioDecoupled) { new MemReq() }.flip + val mem_req = (new ioDecoupled) { new HubMemReq() }.flip val xact_finish = Bool(INPUT) + val p_rep_has_data = Bool(INPUT) + val x_init_has_data = Bool(INPUT) + val p_rep_data_idx = Bits(log2up(NTILES), INPUT) + val x_init_data_idx = Bits(log2up(NTILES), INPUT) + val rep_cnt_dec = Bits(NTILES, INPUT) val busy = Bool(OUTPUT) val addr = Bits(PADDR_BITS, OUTPUT) val tile_id = Bits(TILE_ID_BITS, OUTPUT) val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) val sharer_count = Bits(TILE_ID_BITS, OUTPUT) - val ttype = Bits(TTYPE_BITS, OUTPUT) + val t_type = Bits(TTYPE_BITS, OUTPUT) + val pop_p_rep = Bool(OUTPUT) + val pop_p_rep_data = Bool(OUTPUT) + val send_x_rep_ack = Bool(OUTPUT) } val valid = Reg(resetVal = Bool(false)) val addr = Reg{ Bits() } - val ttype = Reg{ Bits() } + val t_type = Reg{ Bits() } val tile_id = Reg{ Bits() } val tile_xact_id = Reg{ Bits() } val probe_done = Reg{ Bits() } @@ -184,9 +188,9 @@ class CoherenceHubNoDir extends CoherenceHub { def coherenceConflict(addr1: Bits, addr2: Bits): Bool = { addr1(PADDR_BITS-1, OFFSET_BITS) === addr2(PADDR_BITS-1, OFFSET_BITS) } - def getTransactionReplyType(ttype: UFix, count: UFix): Bits = { + def getTransactionReplyType(t_type: UFix, count: UFix): Bits = { val ret = Wire() { Bits(width = TTYPE_BITS) } - switch (ttype) { + switch (t_type) { is(X_READ_SHARED) { ret := Mux(count > UFix(0), X_READ_SHARED, X_READ_EXCLUSIVE) } is(X_READ_EXCLUSIVE) { ret := X_READ_EXCLUSIVE } is(X_READ_UNCACHED) { ret := X_READ_UNCACHED } @@ -205,17 +209,27 @@ class CoherenceHubNoDir extends CoherenceHub { val addr_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } val tile_id_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val tile_xact_id_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } + val t_type_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TTYPE_BITS)} } val sh_count_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val ttype_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TTYPE_BITS)} } - val free_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + val send_x_rep_ack_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + + val do_free_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + val p_rep_has_data_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + val p_rep_data_idx_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=log2up(NTILES))} } + val rep_cnt_dec_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } + for( i <- 0 until NGLOBAL_XACTS) { - busy_arr.write( UFix(i), trackerList(i).io.busy) - addr_arr.write( UFix(i), trackerList(i).io.addr) - tile_id_arr.write( UFix(i), trackerList(i).io.tile_id) - tile_xact_id_arr.write(UFix(i), trackerList(i).io.tile_xact_id) - ttype_arr.write( UFix(i), trackerList(i).io.ttype) - sh_count_arr.write( UFix(i), trackerList(i).io.sharer_count) - trackerList(i).io.xact_finish := free_arr.read(UFix(i)) + busy_arr.write( UFix(i), trackerList(i).io.busy) + addr_arr.write( UFix(i), trackerList(i).io.addr) + tile_id_arr.write( UFix(i), trackerList(i).io.tile_id) + tile_xact_id_arr.write( UFix(i), trackerList(i).io.tile_xact_id) + t_type_arr.write( UFix(i), trackerList(i).io.t_type) + sh_count_arr.write( UFix(i), trackerList(i).io.sharer_count) + send_x_rep_ack_arr.write( UFix(i), trackerList(i).io.send_x_rep_ack) + trackerList(i).io.xact_finish := do_free_arr.read(UFix(i)) + trackerList(i).io.p_rep_has_data := p_rep_has_data_arr.read(UFix(i)) + trackerList(i).io.p_rep_data_idx := p_rep_data_idx_arr.read(UFix(i)) + trackerList(i).io.rep_cnt_dec := rep_cnt_dec_arr.read(UFix(i)) } // Nack conflicting transaction init attempts @@ -231,14 +245,14 @@ class CoherenceHubNoDir extends CoherenceHub { } aborting(j) := (conflicts.orR || busy_arr.flatten().andR) abort.valid := init.valid && aborting - abort.bits.tileTransactionID := init.bits.tileTransactionID + abort.bits.tile_xact_id := init.bits.tile_xact_id init.ready := aborting(j) || initiating(j) } // Free finished transactions for( j <- 0 until NTILES ) { val finish = io.tiles(j).xact_finish - free_arr.write(finish.bits.globalTransactionID, finish.valid) + do_free_arr.write(finish.bits.global_xact_id, finish.valid) finish.ready := Bool(true) } @@ -249,18 +263,55 @@ class CoherenceHubNoDir extends CoherenceHub { val idx = io.mem.resp_tag val readys = Bits(width = NTILES) for( j <- 0 until NTILES ) { - io.tiles(j).xact_rep.bits.ttype := getTransactionReplyType(ttype_arr.read(idx), sh_count_arr.read(idx)) - io.tiles(j).xact_rep.bits.tileTransactionID := tile_xact_id_arr.read(idx) - io.tiles(j).xact_rep.bits.globalTransactionID := idx + io.tiles(j).xact_rep.bits.t_type := getTransactionReplyType(t_type_arr.read(idx), sh_count_arr.read(idx)) + io.tiles(j).xact_rep.bits.tile_xact_id := tile_xact_id_arr.read(idx) + io.tiles(j).xact_rep.bits.global_xact_id := idx io.tiles(j).xact_rep_data.bits.data := io.mem.resp_data readys := Mux(xrep_cnt === UFix(0), io.tiles(j).xact_rep.ready && io.tiles(j).xact_rep_data.ready, io.tiles(j).xact_rep_data.ready) - val this_rep_valid = UFix(j) === tile_id_arr.read(idx) && io.mem.resp_val - io.tiles(j).xact_rep.valid := this_rep_valid && xrep_cnt === UFix(0) - io.tiles(j).xact_rep_data.valid := this_rep_valid + io.tiles(j).xact_rep.valid := (UFix(j) === tile_id_arr.read(idx)) && ((io.mem.resp_val && xrep_cnt === UFix(0)) || send_x_rep_ack_arr.read(idx)) + io.tiles(j).xact_rep_data.valid := (UFix(j) === tile_id_arr.read(idx)) } - // If there were a ready signal due to e.g. intervening network: + // If there were a ready signal due to e.g. intervening network use: //io.mem.resp_rdy := readys(tile_id_arr.read(idx)).xact_rep.ready + // Create an arbiter for the one memory port + // We have to arbitrate between the different trackers' memory requests + // and once we have picked a request, get the right write data + + val mem_req_arb = (new Arbiter(NGLOBAL_XACTS)) { new HubMemReq() } + for( i <- 0 until NGLOBAL_XACTS ) { + mem_req_arb.io.in(i) <> trackerList(i).io.mem_req + } + mem_req_arb.io.out.ready := io.mem.req_rdy + io.mem.req_val := mem_req_arb.io.out.valid + io.mem.req_rw := mem_req_arb.io.out.bits.rw + io.mem.req_tag := mem_req_arb.io.out.bits.tag + io.mem.req_addr := mem_req_arb.io.out.bits.addr + io.mem.req_wdata := MuxLookup(mem_req_arb.io.out.bits.data_idx, + Bits(0, width = MEM_DATA_BITS), + (0 until NTILES).map( j => + UFix(j) -> Mux(mem_req_arb.io.out.bits.is_probe_rep, + io.tiles(j).probe_rep_data.bits.data, + io.tiles(j).xact_init_data.bits.data))) + + for( j <- 0 until NTILES ) { + val p_rep = io.tiles(j).probe_rep + val p_rep_data = io.tiles(j).probe_rep_data + val idx = p_rep.bits.global_xact_id + p_rep_has_data_arr.write(idx, p_rep.valid && p_rep.bits.has_data) + p_rep_data_idx_arr.write(idx, UFix(j)) + p_rep.ready := foldR(trackerList.map(_.io.pop_p_rep))(_ || _) + p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data))(_ || _) + } + for( i <- 0 until NGLOBAL_XACTS ) { + val flags = Bits(width = NTILES) + for( j <- 0 until NTILES) { + val p_rep = io.tiles(j).probe_rep + flags(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) + } + rep_cnt_dec_arr.write(UFix(i), flags) + } + // Pick a single request of these types to process //val xact_init_arb = (new Arbiter(NTILES)) { new TransactionInit() } From ca2e70454e7ad132e02478dac45cbfbb426c1f4d Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Sat, 25 Feb 2012 17:09:26 -0800 Subject: [PATCH 013/374] change package name and sbt project name to rocket --- uncore/coherence.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index d7597cdf..d419dd52 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -1,4 +1,4 @@ -package Top { +package rocket import Chisel._ import Constants._ @@ -319,5 +319,3 @@ class CoherenceHubNoDir extends CoherenceHub { } - -} From e22106af3f3dfc2049d4214a93e60ee5fbd3616a Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Sun, 26 Feb 2012 17:24:08 -0800 Subject: [PATCH 014/374] updating rocket code to lastest version of chisel, passes assembly tests in C++ and Verilog as long as you dont use the vector unit --- uncore/coherence.scala | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index d419dd52..af0d27c8 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -2,7 +2,6 @@ package rocket import Chisel._ import Constants._ -import hwacha.GenArray class HubMemReq extends Bundle { val rw = Bool() @@ -205,18 +204,18 @@ class CoherenceHubNoDir extends CoherenceHub { } val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) - val busy_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } - val addr_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } - val tile_id_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val tile_xact_id_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val t_type_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TTYPE_BITS)} } - val sh_count_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val send_x_rep_ack_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } + val tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } + val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TTYPE_BITS)} } + val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val do_free_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_has_data_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_data_idx_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=log2up(NTILES))} } - val rep_cnt_dec_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } + val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val p_rep_has_data_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val p_rep_data_idx_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=log2up(NTILES))} } + val rep_cnt_dec_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } for( i <- 0 until NGLOBAL_XACTS) { busy_arr.write( UFix(i), trackerList(i).io.busy) @@ -243,7 +242,7 @@ class CoherenceHubNoDir extends CoherenceHub { val t = trackerList(i).io conflicts(i) := t.busy(i) && coherenceConflict(t.addr, init.bits.address) } - aborting(j) := (conflicts.orR || busy_arr.flatten().andR) + aborting(j) := (conflicts.orR || busy_arr.toBits().andR) abort.valid := init.valid && aborting abort.bits.tile_xact_id := init.bits.tile_xact_id init.ready := aborting(j) || initiating(j) From 0fd777f4807f1bf67d69fcbcac2a03aa14124da4 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Sun, 26 Feb 2012 17:24:23 -0800 Subject: [PATCH 015/374] Merge branch 'master' of github.com:ucb-bar/riscv-rocket From b6e6d603cc77e6ab0cd46237922709b61b6ba562 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 26 Feb 2012 00:34:40 -0800 Subject: [PATCH 016/374] xact init transactors in coherence hub --- uncore/coherence.scala | 125 +++++++++++++++++++++++++++-------------- 1 file changed, 82 insertions(+), 43 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index af0d27c8..520f7c05 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -12,6 +12,12 @@ class HubMemReq extends Bundle { val is_probe_rep = Bool() } +class TrackerAllocReq extends Bundle { + val xact_init = new TransactionInit() + val init_tile_id = Bits(width = TILE_ID_BITS) + val data_valid = Bool() +} + class MemData extends Bundle { val data = Bits(width = MEM_DATA_BITS) } @@ -154,6 +160,8 @@ trait FourStateCoherence extends CoherencePolicy { class XactTracker(id: Int) extends Component { val io = new Bundle { val mem_req = (new ioDecoupled) { new HubMemReq() }.flip + val alloc_req = (new ioDecoupled) { new TrackerAllocReq() } + val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) val p_rep_has_data = Bool(INPUT) val x_init_has_data = Bool(INPUT) @@ -162,22 +170,27 @@ class XactTracker(id: Int) extends Component { val rep_cnt_dec = Bits(NTILES, INPUT) val busy = Bool(OUTPUT) val addr = Bits(PADDR_BITS, OUTPUT) - val tile_id = Bits(TILE_ID_BITS, OUTPUT) + val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) val sharer_count = Bits(TILE_ID_BITS, OUTPUT) val t_type = Bits(TTYPE_BITS, OUTPUT) - val pop_p_rep = Bool(OUTPUT) - val pop_p_rep_data = Bool(OUTPUT) + val pop_p_rep = Bits(NTILES, OUTPUT) + val pop_p_rep_data = Bits(NTILES, OUTPUT) + val pop_x_init = Bool(OUTPUT) + val pop_x_init_data = Bool(OUTPUT) val send_x_rep_ack = Bool(OUTPUT) } val valid = Reg(resetVal = Bool(false)) val addr = Reg{ Bits() } val t_type = Reg{ Bits() } - val tile_id = Reg{ Bits() } + val init_tile_id = Reg{ Bits() } val tile_xact_id = Reg{ Bits() } val probe_done = Reg{ Bits() } + //TODO: Decrement the probe count when final data piece is written + // Connent io.mem.ready sig to correct pop* outputs + // P_rep and x_init must be popped on same cycle of receipt } abstract class CoherenceHub extends Component @@ -204,23 +217,23 @@ class CoherenceHubNoDir extends CoherenceHub { } val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) - val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } - val tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TTYPE_BITS)} } - val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val busy_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + val addr_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } + val init_tile_id_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val tile_xact_id_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } + val t_type_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TTYPE_BITS)} } + val sh_count_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val send_x_rep_ack_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } - val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_has_data_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_data_idx_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=log2up(NTILES))} } - val rep_cnt_dec_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } + val do_free_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + val p_rep_has_data_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + val p_rep_data_idx_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=log2up(NTILES))} } + val rep_cnt_dec_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } for( i <- 0 until NGLOBAL_XACTS) { busy_arr.write( UFix(i), trackerList(i).io.busy) addr_arr.write( UFix(i), trackerList(i).io.addr) - tile_id_arr.write( UFix(i), trackerList(i).io.tile_id) + init_tile_id_arr.write( UFix(i), trackerList(i).io.init_tile_id) tile_xact_id_arr.write( UFix(i), trackerList(i).io.tile_xact_id) t_type_arr.write( UFix(i), trackerList(i).io.t_type) sh_count_arr.write( UFix(i), trackerList(i).io.sharer_count) @@ -231,23 +244,6 @@ class CoherenceHubNoDir extends CoherenceHub { trackerList(i).io.rep_cnt_dec := rep_cnt_dec_arr.read(UFix(i)) } - // Nack conflicting transaction init attempts - val aborting = Wire() { Bits(width = NTILES) } - val initiating = Wire() { Bits(width = NTILES) } - for( j <- 0 until NTILES ) { - val init = io.tiles(j).xact_init - val abort = io.tiles(j).xact_abort - val conflicts = Bits(width = NGLOBAL_XACTS) - for( i <- 0 until NGLOBAL_XACTS) { - val t = trackerList(i).io - conflicts(i) := t.busy(i) && coherenceConflict(t.addr, init.bits.address) - } - aborting(j) := (conflicts.orR || busy_arr.toBits().andR) - abort.valid := init.valid && aborting - abort.bits.tile_xact_id := init.bits.tile_xact_id - init.ready := aborting(j) || initiating(j) - } - // Free finished transactions for( j <- 0 until NTILES ) { val finish = io.tiles(j).xact_finish @@ -255,6 +251,7 @@ class CoherenceHubNoDir extends CoherenceHub { finish.ready := Bool(true) } + // Reply to initial requestor // Forward memory responses from mem to tile val xrep_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) val xrep_cnt_next = xrep_cnt + UFix(1) @@ -267,11 +264,11 @@ class CoherenceHubNoDir extends CoherenceHub { io.tiles(j).xact_rep.bits.global_xact_id := idx io.tiles(j).xact_rep_data.bits.data := io.mem.resp_data readys := Mux(xrep_cnt === UFix(0), io.tiles(j).xact_rep.ready && io.tiles(j).xact_rep_data.ready, io.tiles(j).xact_rep_data.ready) - io.tiles(j).xact_rep.valid := (UFix(j) === tile_id_arr.read(idx)) && ((io.mem.resp_val && xrep_cnt === UFix(0)) || send_x_rep_ack_arr.read(idx)) - io.tiles(j).xact_rep_data.valid := (UFix(j) === tile_id_arr.read(idx)) + io.tiles(j).xact_rep.valid := (UFix(j) === init_tile_id_arr.read(idx)) && ((io.mem.resp_val && xrep_cnt === UFix(0)) || send_x_rep_ack_arr.read(idx)) + io.tiles(j).xact_rep_data.valid := (UFix(j) === init_tile_id_arr.read(idx)) } // If there were a ready signal due to e.g. intervening network use: - //io.mem.resp_rdy := readys(tile_id_arr.read(idx)).xact_rep.ready + //io.mem.resp_rdy := readys(init_tile_id_arr.read(idx)).xact_rep.ready // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests @@ -293,28 +290,70 @@ class CoherenceHubNoDir extends CoherenceHub { io.tiles(j).probe_rep_data.bits.data, io.tiles(j).xact_init_data.bits.data))) + // Handle probe replies, which may or may not have data for( j <- 0 until NTILES ) { val p_rep = io.tiles(j).probe_rep val p_rep_data = io.tiles(j).probe_rep_data val idx = p_rep.bits.global_xact_id - p_rep_has_data_arr.write(idx, p_rep.valid && p_rep.bits.has_data) + p_rep_has_data_arr.write(idx, p_rep.valid && p_rep.bits.has_data && p_rep_data.valid) p_rep_data_idx_arr.write(idx, UFix(j)) - p_rep.ready := foldR(trackerList.map(_.io.pop_p_rep))(_ || _) - p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data))(_ || _) + p_rep.ready := foldR(trackerList.map(_.io.pop_p_rep(j)))(_ || _) + p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) } for( i <- 0 until NGLOBAL_XACTS ) { val flags = Bits(width = NTILES) for( j <- 0 until NTILES) { val p_rep = io.tiles(j).probe_rep - flags(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) + flags(j) := p_rep.valid && !p_rep.bits.has_data && (p_rep.bits.global_xact_id === UFix(i)) } rep_cnt_dec_arr.write(UFix(i), flags) } + // Nack conflicting transaction init attempts + val aborting = Wire() { Bits(width = NTILES) } + for( j <- 0 until NTILES ) { + val x_init = io.tiles(j).xact_init + val x_abort = io.tiles(j).xact_abort + val conflicts = Bits(width = NGLOBAL_XACTS) + for( i <- 0 until NGLOBAL_XACTS) { + val t = trackerList(i).io + conflicts(i) := t.busy(i) && coherenceConflict(t.addr, x_init.bits.address) && + !(x_init.bits.has_data && (UFix(j) === t.init_tile_id)) + // Don't abort writebacks stalled on mem. + // TODO: This assumes overlapped writeback init reqs to + // the same addr will never be issued; is this ok? + } + x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id + val want_to_abort = conflicts.orR || busy_arr.flatten().andR + x_abort.valid := want_to_abort && x_init.valid + aborting(j) := want_to_abort && x_abort.ready + } + + // Handle transaction initiation requests + // Only one allocation per cycle + // Init requests may or may not have data + val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } + val init_arb = (new Arbiter(NTILES)) { new TrackerAllocReq() } + for( i <- 0 until NGLOBAL_XACTS ) { + alloc_arb.io.in(i).valid := !trackerList(i).io.busy + trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready + trackerList(i).io.alloc_req.bits := init_arb.io.out.bits + trackerList(i).io.alloc_req.valid := init_arb.io.out.valid + } - // Pick a single request of these types to process - //val xact_init_arb = (new Arbiter(NTILES)) { new TransactionInit() } - //val probe_reply_arb = (new Arbiter(NTILES)) { new ProbeReply() } + for( j <- 0 until NTILES ) { + val x_init = io.tiles(j).xact_init + val x_init_data = io.tiles(j).xact_init_data + init_arb.io.in(j).valid := x_init.valid + init_arb.io.in(j).bits.xact_init := x_init.bits + init_arb.io.in(j).bits.init_tile_id := UFix(j) + init_arb.io.in(j).bits.data_valid := x_init_data.valid + x_init.ready := aborting(j) || foldR(trackerList.map(_.io.pop_x_init && init_arb.io.out.bits.init_tile_id === UFix(j)))(_||_) + x_init_data.ready := aborting(j) || foldR(trackerList.map(_.io.pop_x_init_data && init_arb.io.out.bits.init_tile_id === UFix(j)))(_||_) + } + + alloc_arb.io.out.ready := init_arb.io.out.valid && !busy_arr.flatten().andR && + !foldR(trackerList.map(t => t.io.busy && coherenceConflict(t.io.addr, init_arb.io.out.bits.xact_init.address)))(_||_) } From 70cdf869ac94af7c50405218cb0cf1a87cc76f0c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 26 Feb 2012 00:53:07 -0800 Subject: [PATCH 017/374] probe req transactors in coherence hub --- uncore/coherence.scala | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 520f7c05..8cc616de 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -161,6 +161,7 @@ class XactTracker(id: Int) extends Component { val io = new Bundle { val mem_req = (new ioDecoupled) { new HubMemReq() }.flip val alloc_req = (new ioDecoupled) { new TrackerAllocReq() } + val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) val p_rep_has_data = Bool(INPUT) @@ -174,6 +175,7 @@ class XactTracker(id: Int) extends Component { val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) val sharer_count = Bits(TILE_ID_BITS, OUTPUT) val t_type = Bits(TTYPE_BITS, OUTPUT) + val push_p_req = Bits(NTILES, OUTPUT) val pop_p_rep = Bits(NTILES, OUTPUT) val pop_p_rep_data = Bits(NTILES, OUTPUT) val pop_x_init = Bool(OUTPUT) @@ -356,4 +358,17 @@ class CoherenceHubNoDir extends CoherenceHub { !foldR(trackerList.map(t => t.io.busy && coherenceConflict(t.io.addr, init_arb.io.out.bits.xact_init.address)))(_||_) + // Handle probe request generation + // Must arbitrate for each request port + for( j <- 0 until NTILES ) { + val p_req_arb = (new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() } + for( i <- 0 until NGLOBAL_XACTS ) { + val t = trackerList(i).io + p_req_arb.io.in(i).bits := t.probe_req.bits + p_req_arb.io.in(i).ready := t.probe_req.ready + p_req_arb.io.in(i).valid := t.probe_req.valid && t.push_p_req(j) + } + p_req_arb.io.out <> io.tiles(j).probe_req + } + } From 3393dc2d31aaa857daf3d604c60e8d95e5493450 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 27 Feb 2012 11:26:18 -0800 Subject: [PATCH 018/374] Added probe_req ready sigs, GenArray to Vec --- uncore/coherence.scala | 68 ++++++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 8cc616de..8aff1592 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -168,7 +168,8 @@ class XactTracker(id: Int) extends Component { val x_init_has_data = Bool(INPUT) val p_rep_data_idx = Bits(log2up(NTILES), INPUT) val x_init_data_idx = Bits(log2up(NTILES), INPUT) - val rep_cnt_dec = Bits(NTILES, INPUT) + val p_rep_cnt_dec = Bits(NTILES, INPUT) + val p_req_cnt_inc = Bits(NTILES, INPUT) val busy = Bool(OUTPUT) val addr = Bits(PADDR_BITS, OUTPUT) val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) @@ -218,32 +219,35 @@ class CoherenceHubNoDir extends CoherenceHub { val mem = new ioDCache().flip } - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) - val busy_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } - val addr_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } - val init_tile_id_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val tile_xact_id_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val t_type_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TTYPE_BITS)} } - val sh_count_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val send_x_rep_ack_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) - val do_free_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_has_data_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_data_idx_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=log2up(NTILES))} } - val rep_cnt_dec_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } + val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } + val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } + val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TTYPE_BITS)} } + val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + + val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val p_rep_has_data_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val p_rep_data_idx_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=log2up(NTILES))} } + val p_rep_cnt_dec_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } + val p_req_cnt_inc_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } for( i <- 0 until NGLOBAL_XACTS) { - busy_arr.write( UFix(i), trackerList(i).io.busy) - addr_arr.write( UFix(i), trackerList(i).io.addr) - init_tile_id_arr.write( UFix(i), trackerList(i).io.init_tile_id) - tile_xact_id_arr.write( UFix(i), trackerList(i).io.tile_xact_id) + busy_arr.write( UFix(i), trackerList(i).io.busy) + addr_arr.write( UFix(i), trackerList(i).io.addr) + init_tile_id_arr.write( UFix(i), trackerList(i).io.init_tile_id) + tile_xact_id_arr.write( UFix(i), trackerList(i).io.tile_xact_id) t_type_arr.write( UFix(i), trackerList(i).io.t_type) - sh_count_arr.write( UFix(i), trackerList(i).io.sharer_count) - send_x_rep_ack_arr.write( UFix(i), trackerList(i).io.send_x_rep_ack) - trackerList(i).io.xact_finish := do_free_arr.read(UFix(i)) + sh_count_arr.write( UFix(i), trackerList(i).io.sharer_count) + send_x_rep_ack_arr.write(UFix(i), trackerList(i).io.send_x_rep_ack) + trackerList(i).io.xact_finish := do_free_arr.read(UFix(i)) trackerList(i).io.p_rep_has_data := p_rep_has_data_arr.read(UFix(i)) trackerList(i).io.p_rep_data_idx := p_rep_data_idx_arr.read(UFix(i)) - trackerList(i).io.rep_cnt_dec := rep_cnt_dec_arr.read(UFix(i)) + trackerList(i).io.p_rep_cnt_dec := p_rep_cnt_dec_arr.read(UFix(i)) + trackerList(i).io.p_req_cnt_inc := p_req_cnt_inc_arr.read(UFix(i)) } // Free finished transactions @@ -308,7 +312,7 @@ class CoherenceHubNoDir extends CoherenceHub { val p_rep = io.tiles(j).probe_rep flags(j) := p_rep.valid && !p_rep.bits.has_data && (p_rep.bits.global_xact_id === UFix(i)) } - rep_cnt_dec_arr.write(UFix(i), flags) + p_rep_cnt_dec_arr.write(UFix(i), flags) } // Nack conflicting transaction init attempts @@ -326,7 +330,7 @@ class CoherenceHubNoDir extends CoherenceHub { // the same addr will never be issued; is this ok? } x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id - val want_to_abort = conflicts.orR || busy_arr.flatten().andR + val want_to_abort = conflicts.orR || busy_arr.toBits.andR x_abort.valid := want_to_abort && x_init.valid aborting(j) := want_to_abort && x_abort.ready } @@ -354,21 +358,27 @@ class CoherenceHubNoDir extends CoherenceHub { x_init_data.ready := aborting(j) || foldR(trackerList.map(_.io.pop_x_init_data && init_arb.io.out.bits.init_tile_id === UFix(j)))(_||_) } - alloc_arb.io.out.ready := init_arb.io.out.valid && !busy_arr.flatten().andR && + alloc_arb.io.out.ready := init_arb.io.out.valid && !busy_arr.toBits.andR && !foldR(trackerList.map(t => t.io.busy && coherenceConflict(t.io.addr, init_arb.io.out.bits.xact_init.address)))(_||_) // Handle probe request generation // Must arbitrate for each request port + val p_req_arb_arr = List.fill(NTILES)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) for( j <- 0 until NTILES ) { - val p_req_arb = (new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() } for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io - p_req_arb.io.in(i).bits := t.probe_req.bits - p_req_arb.io.in(i).ready := t.probe_req.ready - p_req_arb.io.in(i).valid := t.probe_req.valid && t.push_p_req(j) + p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits + p_req_arb_arr(j).io.in(i).valid := t.probe_req.valid && t.push_p_req(j) } - p_req_arb.io.out <> io.tiles(j).probe_req + p_req_arb_arr(j).io.out <> io.tiles(j).probe_req + } + for( i <- 0 until NGLOBAL_XACTS ) { + val flags = Bits(width = NTILES) + for( j <- 0 until NTILES ) { + flags(j) := p_req_arb_arr(j).io.in(i).ready + } + p_rep_cnt_dec_arr.write(UFix(i), flags) } } From 7d331858e25c840e19fb745b01b0465137230901 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 27 Feb 2012 18:36:09 -0800 Subject: [PATCH 019/374] replace ioDCache with ioMem --- uncore/coherence.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 8aff1592..d5a5e414 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -216,7 +216,7 @@ class CoherenceHubNoDir extends CoherenceHub { val io = new Bundle { val tiles = Vec(NTILES) { new ioTileLink() } - val mem = new ioDCache().flip + val mem = new ioMem } val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) From ffc7652e0166b38b4db2612e93358dd267e22206 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 27 Feb 2012 19:10:15 -0800 Subject: [PATCH 020/374] Null coherence hub. Begin work on internal tracker logic --- uncore/coherence.scala | 163 +++++++++++++++++++++++++++++++++-------- 1 file changed, 133 insertions(+), 30 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index d5a5e414..20c95e78 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -157,46 +157,149 @@ trait FourStateCoherence extends CoherencePolicy { } } -class XactTracker(id: Int) extends Component { +class XactTracker(id: Int) extends Component with CoherencePolicy { val io = new Bundle { - val mem_req = (new ioDecoupled) { new HubMemReq() }.flip - val alloc_req = (new ioDecoupled) { new TrackerAllocReq() } - val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip - val can_alloc = Bool(INPUT) - val xact_finish = Bool(INPUT) - val p_rep_has_data = Bool(INPUT) - val x_init_has_data = Bool(INPUT) - val p_rep_data_idx = Bits(log2up(NTILES), INPUT) - val x_init_data_idx = Bits(log2up(NTILES), INPUT) - val p_rep_cnt_dec = Bits(NTILES, INPUT) - val p_req_cnt_inc = Bits(NTILES, INPUT) - val busy = Bool(OUTPUT) - val addr = Bits(PADDR_BITS, OUTPUT) - val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) - val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) - val sharer_count = Bits(TILE_ID_BITS, OUTPUT) - val t_type = Bits(TTYPE_BITS, OUTPUT) - val push_p_req = Bits(NTILES, OUTPUT) - val pop_p_rep = Bits(NTILES, OUTPUT) - val pop_p_rep_data = Bits(NTILES, OUTPUT) - val pop_x_init = Bool(OUTPUT) + val alloc_req = (new ioDecoupled) { new TrackerAllocReq() } + val can_alloc = Bool(INPUT) + val xact_finish = Bool(INPUT) + val p_rep_has_data = Bool(INPUT) + val p_rep_data_idx = Bits(log2up(NTILES), INPUT) + val p_rep_cnt_dec = Bits(NTILES, INPUT) + val p_req_cnt_inc = Bits(NTILES, INPUT) + + val mem_req = (new ioDecoupled) { new HubMemReq() }.flip + val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip + val busy = Bool(OUTPUT) + val addr = Bits(PADDR_BITS, OUTPUT) + val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) + val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) + val sharer_count = Bits(TILE_ID_BITS, OUTPUT) + val t_type = Bits(TTYPE_BITS, OUTPUT) + val push_p_req = Bits(NTILES, OUTPUT) + val pop_p_rep = Bits(NTILES, OUTPUT) + val pop_p_rep_data = Bits(NTILES, OUTPUT) + val pop_x_init = Bool(OUTPUT) val pop_x_init_data = Bool(OUTPUT) - val send_x_rep_ack = Bool(OUTPUT) + val send_x_rep_ack = Bool(OUTPUT) } - val valid = Reg(resetVal = Bool(false)) - val addr = Reg{ Bits() } - val t_type = Reg{ Bits() } - val init_tile_id = Reg{ Bits() } - val tile_xact_id = Reg{ Bits() } - val probe_done = Reg{ Bits() } + val s_idle :: s_mem_r :: s_mem_w :: mem_wr :: s_probe :: Nil = Enum(5){ UFix() } + val state = Reg(resetVal = s_idle) + val addr_ = Reg{ Bits() } + val t_type_ = Reg{ Bits() } + val init_tile_id_ = Reg{ Bits() } + val tile_xact_id_ = Reg{ Bits() } + val probe_done = Reg{ Bits() } + val mem_count = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + + io.addr := addr_ + io.init_tile_id := init_tile_id_ + io.tile_xact_id := tile_xact_id_ + io.sharer_count := UFix(NTILES) // TODO: Broadcast only + io.t_type := t_type_ + +/* +class HubMemReq extends Bundle { + val rw = Bool() + val addr = UFix(width = PADDR_BITS-OFFSET_BITS) + val tag = Bits(width = GLOBAL_XACT_ID_BITS) + // Figure out which data-in port to pull from + val data_idx = Bits(width = TILE_ID_BITS) + val is_probe_rep = Bool() +} + +class TrackerAllocReq extends Bundle { + val xact_init = new TransactionInit() + val t_type = Bits(width = TTYPE_BITS) + val has_data = Bool() + val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) + val address = Bits(width = PADDR_BITS) + val init_tile_id = Bits(width = TILE_ID_BITS) + val data_valid = Bool() +*/ +/* + when( alloc_req.valid && can_alloc ) { + valid := Bool(true) + addr := alloc_req.bits.xact_init.address + t_type := alloc_req.bits.xact_init.t_type + init_tile_id := alloc_req.bits.init_tile_id + tile_xact_id := alloc_req.bits.xact_init.tile_xact_id + [counter] := REFILL_CYCLES-1 if alloc_req.bits.xact_init.has_data else 0 + } + when ( alloc_req.bits.data_valid ) { + io.mem_req.valid := + io.mem_req.bits.rw := + io.mem_req.bits.addr := + io.mem_req.bits.tag := + io.mem_req.bits.data_idx := + io.mem_req.bits.is_probe_rep := + := io.mem.ready + } + when( p_rep_has_data ) { + io.mem_req.valid := + io.mem_req.bits.rw := + io.mem_req.bits.addr := + io.mem_req.bits.tag := + io.mem_req.bits.data_idx := + io.mem_req.bits.is_probe_rep := + := io.mem.ready + + } + + val mem_req = (new ioDecoupled) { new HubMemReq() }.flip + val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip + push_p_req = Bits(0, width = NTILES) + pop_p_rep = Bits(0, width = NTILES) + pop_p_rep_data = Bits(0, width = NTILES) + pop_x_init = Bool(false) + pop_x_init_data = Bool(false) + send_x_rep_ack = Bool(false) + + + } +*/ //TODO: Decrement the probe count when final data piece is written // Connent io.mem.ready sig to correct pop* outputs // P_rep and x_init must be popped on same cycle of receipt } -abstract class CoherenceHub extends Component +abstract class CoherenceHub extends Component with CoherencePolicy + +class CoherenceHubNull extends Component { + val io = new Bundle { + val tile = new ioTileLink() + val mem = new ioMem() + } + + val x_init = io.tile.xact_init + val is_write = x_init.bits.t_type === X_WRITE_UNCACHED + x_init.ready := io.mem.req_rdy + io.mem.req_val := x_init.valid + io.mem.req_rw := is_write + io.mem.req_tag := x_init.bits.tile_xact_id + io.mem.req_addr := x_init.bits.address + + val x_rep = io.tile.xact_rep + x_rep.bits.t_type := Bits(width = TTYPE_BITS) + x_rep.bits.has_data := !is_write + x_rep.bits.tile_xact_id := Mux(is_write, x_init.bits.tile_xact_id, io.mem.resp_tag) + x_rep.bits.global_xact_id := UFix(0) // don't care + x_rep.valid := io.mem.resp_val + + //TODO: + val x_init_data = io.tile.xact_init_data + val x_rep_data = io.tile.xact_rep_data + x_init_data.ready := io.mem.req_rdy + io.mem.req_wdata := x_init_data.bits.data + x_rep_data.bits.data := io.mem.resp_data + x_rep_data.valid := io.mem.resp_val + // Should be: + //io.mem.req_data <> x_init_data + //x_rep_data <> io.mem.resp_data + +} + class CoherenceHubNoDir extends CoherenceHub { From e15284d22ce5dcef9a2a1af46e4de9c81d0a3566 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 28 Feb 2012 17:33:06 -0800 Subject: [PATCH 021/374] Added temporary ioMemHub and made coherence hub implementations depend on it rather than ioMem --- uncore/coherence.scala | 281 +++++++++++++++++++++++------------------ 1 file changed, 157 insertions(+), 124 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 20c95e78..11c764ec 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -3,13 +3,39 @@ package rocket import Chisel._ import Constants._ +class MemData extends Bundle { + val data = Bits(width = MEM_DATA_BITS) +} + +class MemReqCmd() extends Bundle +{ + val rw = Bool() + val addr = UFix(PADDR_BITS - OFFSET_BITS) + val tag = Bits(MEM_TAG_BITS) +} + +class MemResp () extends Bundle +{ + val tag = Bits(MEM_TAG_BITS) + val data = Bits(width = MEM_DATA_BITS) + val valid = Bool() +} + +class ioMemHub() extends Bundle +{ + val req_cmd = (new ioDecoupled) { new MemReqCmd() }.flip + val req_data = (new ioDecoupled) { new MemData() }.flip + val resp = new MemResp() +} + class HubMemReq extends Bundle { - val rw = Bool() - val addr = UFix(width = PADDR_BITS-OFFSET_BITS) - val tag = Bits(width = GLOBAL_XACT_ID_BITS) - // Figure out which data-in port to pull from - val data_idx = Bits(width = TILE_ID_BITS) - val is_probe_rep = Bool() + val req_cmd = (new ioDecoupled) { new MemReqCmd() } + val req_data = (new ioDecoupled) { new MemData() } +} + +class HubProbeRep extends Bundle { + val reply = (new ioDecoupled) { new ProbeReply } + val data_idx = Bits(width = log2up(NTILES)) } class TrackerAllocReq extends Bundle { @@ -18,9 +44,6 @@ class TrackerAllocReq extends Bundle { val data_valid = Bool() } -class MemData extends Bundle { - val data = Bits(width = MEM_DATA_BITS) -} class TransactionInit extends Bundle { val t_type = Bits(width = TTYPE_BITS) @@ -49,15 +72,13 @@ class ProbeReply extends Bundle { class ProbeReplyData extends MemData -class TransactionReply extends Bundle { +class TransactionReply extends MemData { val t_type = Bits(width = TTYPE_BITS) val has_data = Bool() val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } -class TransactionReplyData extends MemData - class TransactionFinish extends Bundle { val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } @@ -70,7 +91,6 @@ class ioTileLink extends Bundle { val probe_rep = (new ioDecoupled) { new ProbeReply() }.flip val probe_rep_data = (new ioDecoupled) { new ProbeReplyData() }.flip val xact_rep = (new ioDecoupled) { new TransactionReply() } - val xact_rep_data = (new ioDecoupled) { new TransactionReplyData() } val xact_finish = (new ioDecoupled) { new TransactionFinish() }.flip } @@ -160,18 +180,21 @@ trait FourStateCoherence extends CoherencePolicy { class XactTracker(id: Int) extends Component with CoherencePolicy { val io = new Bundle { val alloc_req = (new ioDecoupled) { new TrackerAllocReq() } + val probe_rep = (new ioDecoupled) { new HubProbeRep() } val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) val p_rep_has_data = Bool(INPUT) - val p_rep_data_idx = Bits(log2up(NTILES), INPUT) val p_rep_cnt_dec = Bits(NTILES, INPUT) val p_req_cnt_inc = Bits(NTILES, INPUT) + val p_rep_data = (new ioDecoupled) { new ProbeReplyData() } + val x_init_data = (new ioDecoupled) { new TransactionInitData() } val mem_req = (new ioDecoupled) { new HubMemReq() }.flip val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip val busy = Bool(OUTPUT) val addr = Bits(PADDR_BITS, OUTPUT) val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) + val p_rep_tile_id = Bits(log2up(NTILES), INPUT) val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) val sharer_count = Bits(TILE_ID_BITS, OUTPUT) val t_type = Bits(TTYPE_BITS, OUTPUT) @@ -183,8 +206,14 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { val send_x_rep_ack = Bool(OUTPUT) } - - val s_idle :: s_mem_r :: s_mem_w :: mem_wr :: s_probe :: Nil = Enum(5){ UFix() } + def sendProbeReqType(t_type: UFix, global_state: UFix): UFix = { + MuxCase(P_COPY, Array((t_type === X_READ_SHARED) -> P_DOWNGRADE, + (t_type === X_READ_EXCLUSIVE) -> P_INVALIDATE, + (t_type === X_READ_UNCACHED) -> P_COPY, + (t_type === X_WRITE_UNCACHED) -> P_INVALIDATE)) + } + + val s_idle :: s_mem_r :: s_mem_w :: s_mem_wr :: s_probe :: s_busy :: Nil = Enum(6){ UFix() } val state = Reg(resetVal = s_idle) val addr_ = Reg{ Bits() } val t_type_ = Reg{ Bits() } @@ -192,73 +221,99 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { val tile_xact_id_ = Reg{ Bits() } val probe_done = Reg{ Bits() } val mem_count = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val p_rep_count = Reg(resetVal = UFix(0, width = log2up(NTILES))) + val p_req_flags = Reg(resetVal = UFix(0, width = NTILES)) + val p_rep_data_idx_ = Reg{ Bits() } + val x_init_data_needs_wb = Reg{ Bool() } + val p_rep_data_needs_wb = Reg{ Bool() } + io.busy := state != s_idle io.addr := addr_ io.init_tile_id := init_tile_id_ io.tile_xact_id := tile_xact_id_ io.sharer_count := UFix(NTILES) // TODO: Broadcast only io.t_type := t_type_ -/* -class HubMemReq extends Bundle { - val rw = Bool() - val addr = UFix(width = PADDR_BITS-OFFSET_BITS) - val tag = Bits(width = GLOBAL_XACT_ID_BITS) - // Figure out which data-in port to pull from - val data_idx = Bits(width = TILE_ID_BITS) - val is_probe_rep = Bool() -} - -class TrackerAllocReq extends Bundle { - val xact_init = new TransactionInit() - val t_type = Bits(width = TTYPE_BITS) - val has_data = Bool() - val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) - val address = Bits(width = PADDR_BITS) - val init_tile_id = Bits(width = TILE_ID_BITS) - val data_valid = Bool() -*/ -/* - when( alloc_req.valid && can_alloc ) { - valid := Bool(true) - addr := alloc_req.bits.xact_init.address - t_type := alloc_req.bits.xact_init.t_type - init_tile_id := alloc_req.bits.init_tile_id - tile_xact_id := alloc_req.bits.xact_init.tile_xact_id - [counter] := REFILL_CYCLES-1 if alloc_req.bits.xact_init.has_data else 0 - } - when ( alloc_req.bits.data_valid ) { - io.mem_req.valid := - io.mem_req.bits.rw := - io.mem_req.bits.addr := - io.mem_req.bits.tag := - io.mem_req.bits.data_idx := - io.mem_req.bits.is_probe_rep := - := io.mem.ready - } - when( p_rep_has_data ) { - io.mem_req.valid := - io.mem_req.bits.rw := - io.mem_req.bits.addr := - io.mem_req.bits.tag := - io.mem_req.bits.data_idx := - io.mem_req.bits.is_probe_rep := - := io.mem.ready + io.mem_req.valid := Bool(false) + io.mem_req.bits.req_cmd.bits.rw := state === s_mem_w || state === s_mem_wr + io.mem_req.bits.req_cmd.bits.addr := addr_ + io.mem_req.bits.req_cmd.bits.tag := UFix(id) + // := io.mem.ready //sent mem req + io.probe_req.valid := Bool(false) + io.probe_req.bits.p_type := sendProbeReqType(t_type_, UFix(0)) + io.probe_req.bits.global_xact_id := UFix(id) + io.probe_req.bits.address := addr_ + // := io.probe_req.ready //got through arbiter ---- p_rep_dec_arr + io.push_p_req := Bits(0, width = NTILES) + io.pop_p_rep := Bits(0, width = NTILES) + io.pop_p_rep_data := Bits(0, width = NTILES) + io.pop_x_init := Bool(false) + io.pop_x_init_data := Bool(false) + io.send_x_rep_ack := Bool(false) + switch (state) { + is(s_idle) { + when( io.alloc_req.valid && io.can_alloc ) { + addr_ := io.alloc_req.bits.xact_init.address + t_type_ := io.alloc_req.bits.xact_init.t_type + init_tile_id_ := io.alloc_req.bits.init_tile_id + tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id + x_init_data_needs_wb := io.alloc_req.bits.xact_init.has_data + p_rep_count := UFix(NTILES) + p_req_flags := ~Bits(0, width = NTILES) + state := s_probe + io.pop_x_init := Bool(true) + } + } + is(s_mem_r) { + io.mem_req.valid := Bool(true) + when(io.mem_req.ready) { state := s_busy } + } + is(s_mem_w) { + io.mem_req.valid := Bool(true) + when(io.mem_req.ready) { state := s_busy } + } + is(s_mem_wr) { + when(io.probe_rep.bits.reply.bits.has_data) { + //io.pop_p_rep(p_rep_data_idx) := io.mem_req_rdy + //io.pop_p_rep_data(p_rep_data_idx) := io.mem_req_rdy //TODO + } . otherwise { + //io.pop_x_init := io.mem_req_rdy + //io.pop_x_init_data := io.mem_req_rdy + } + io.mem_req.valid := Bool(true) + when(io.mem_req.ready) { state := s_mem_r } + } + is(s_probe) { + when(p_req_flags.orR) { + io.push_p_req := p_req_flags + io.probe_req.valid := Bool(true) + } + when(io.p_req_cnt_inc.orR) { + p_req_flags := p_req_flags & ~io.p_req_cnt_inc // unflag sent reqs + } + val p_rep_has_data = Bool(INPUT) + val p_rep_data_idx = Bits(log2up(NTILES), INPUT) + val p_rep_cnt_dec = Bits(NTILES, INPUT) + when(io.p_rep_cnt_dec.orR) { + val p_rep_count_next = p_rep_count - PopCount(io.p_rep_cnt_dec) + p_rep_count := p_rep_count_next + when(p_rep_count_next === UFix(0)) { + state := s_busy //TODO: XXXXXXXXXX + } + } + when(p_rep_has_data) { + p_rep_data_needs_wb := Bool(true) + p_rep_data_idx_ := p_rep_data_idx + } + } + is(s_busy) { + when (io.xact_finish) { + state := s_idle + } + } } - val mem_req = (new ioDecoupled) { new HubMemReq() }.flip - val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip - push_p_req = Bits(0, width = NTILES) - pop_p_rep = Bits(0, width = NTILES) - pop_p_rep_data = Bits(0, width = NTILES) - pop_x_init = Bool(false) - pop_x_init_data = Bool(false) - send_x_rep_ack = Bool(false) - - - } -*/ //TODO: Decrement the probe count when final data piece is written // Connent io.mem.ready sig to correct pop* outputs // P_rep and x_init must be popped on same cycle of receipt @@ -269,35 +324,24 @@ abstract class CoherenceHub extends Component with CoherencePolicy class CoherenceHubNull extends Component { val io = new Bundle { val tile = new ioTileLink() - val mem = new ioMem() + val mem = new ioMemHub() } val x_init = io.tile.xact_init val is_write = x_init.bits.t_type === X_WRITE_UNCACHED - x_init.ready := io.mem.req_rdy - io.mem.req_val := x_init.valid - io.mem.req_rw := is_write - io.mem.req_tag := x_init.bits.tile_xact_id - io.mem.req_addr := x_init.bits.address + x_init.ready := io.mem.req_cmd.ready + io.mem.req_cmd.valid := x_init.valid + io.mem.req_cmd.bits.rw := is_write + io.mem.req_cmd.bits.tag := x_init.bits.tile_xact_id + io.mem.req_cmd.bits.addr := x_init.bits.address + io.mem.req_data <> io.tile.xact_init_data val x_rep = io.tile.xact_rep - x_rep.bits.t_type := Bits(width = TTYPE_BITS) - x_rep.bits.has_data := !is_write - x_rep.bits.tile_xact_id := Mux(is_write, x_init.bits.tile_xact_id, io.mem.resp_tag) + x_rep.bits.t_type := X_READ_EXCLUSIVE + x_rep.bits.tile_xact_id := Mux(is_write, x_init.bits.tile_xact_id, io.mem.resp.tag) x_rep.bits.global_xact_id := UFix(0) // don't care - x_rep.valid := io.mem.resp_val - - //TODO: - val x_init_data = io.tile.xact_init_data - val x_rep_data = io.tile.xact_rep_data - x_init_data.ready := io.mem.req_rdy - io.mem.req_wdata := x_init_data.bits.data - x_rep_data.bits.data := io.mem.resp_data - x_rep_data.valid := io.mem.resp_val - // Should be: - //io.mem.req_data <> x_init_data - //x_rep_data <> io.mem.resp_data - + x_rep.bits.data := io.mem.resp.data + x_rep.valid := io.mem.resp.valid } @@ -319,7 +363,7 @@ class CoherenceHubNoDir extends CoherenceHub { val io = new Bundle { val tiles = Vec(NTILES) { new ioTileLink() } - val mem = new ioMem + val mem = new ioMemHub } val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) @@ -333,8 +377,6 @@ class CoherenceHubNoDir extends CoherenceHub { val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_has_data_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_data_idx_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=log2up(NTILES))} } val p_rep_cnt_dec_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } val p_req_cnt_inc_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } @@ -347,8 +389,6 @@ class CoherenceHubNoDir extends CoherenceHub { sh_count_arr.write( UFix(i), trackerList(i).io.sharer_count) send_x_rep_ack_arr.write(UFix(i), trackerList(i).io.send_x_rep_ack) trackerList(i).io.xact_finish := do_free_arr.read(UFix(i)) - trackerList(i).io.p_rep_has_data := p_rep_has_data_arr.read(UFix(i)) - trackerList(i).io.p_rep_data_idx := p_rep_data_idx_arr.read(UFix(i)) trackerList(i).io.p_rep_cnt_dec := p_rep_cnt_dec_arr.read(UFix(i)) trackerList(i).io.p_req_cnt_inc := p_req_cnt_inc_arr.read(UFix(i)) } @@ -362,22 +402,16 @@ class CoherenceHubNoDir extends CoherenceHub { // Reply to initial requestor // Forward memory responses from mem to tile - val xrep_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) - val xrep_cnt_next = xrep_cnt + UFix(1) - when (io.mem.resp_val) { xrep_cnt := xrep_cnt_next } - val idx = io.mem.resp_tag - val readys = Bits(width = NTILES) + val idx = io.mem.resp.tag for( j <- 0 until NTILES ) { io.tiles(j).xact_rep.bits.t_type := getTransactionReplyType(t_type_arr.read(idx), sh_count_arr.read(idx)) io.tiles(j).xact_rep.bits.tile_xact_id := tile_xact_id_arr.read(idx) io.tiles(j).xact_rep.bits.global_xact_id := idx - io.tiles(j).xact_rep_data.bits.data := io.mem.resp_data - readys := Mux(xrep_cnt === UFix(0), io.tiles(j).xact_rep.ready && io.tiles(j).xact_rep_data.ready, io.tiles(j).xact_rep_data.ready) - io.tiles(j).xact_rep.valid := (UFix(j) === init_tile_id_arr.read(idx)) && ((io.mem.resp_val && xrep_cnt === UFix(0)) || send_x_rep_ack_arr.read(idx)) - io.tiles(j).xact_rep_data.valid := (UFix(j) === init_tile_id_arr.read(idx)) + io.tiles(j).xact_rep.bits.data := io.mem.resp.data + io.tiles(j).xact_rep.valid := (UFix(j) === init_tile_id_arr.read(idx)) && (io.mem.resp.valid || send_x_rep_ack_arr.read(idx)) } // If there were a ready signal due to e.g. intervening network use: - //io.mem.resp_rdy := readys(init_tile_id_arr.read(idx)).xact_rep.ready + //io.mem.resp.ready := io.tiles(init_tile_id_arr.read(idx)).xact_rep.ready // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests @@ -387,33 +421,30 @@ class CoherenceHubNoDir extends CoherenceHub { for( i <- 0 until NGLOBAL_XACTS ) { mem_req_arb.io.in(i) <> trackerList(i).io.mem_req } - mem_req_arb.io.out.ready := io.mem.req_rdy - io.mem.req_val := mem_req_arb.io.out.valid - io.mem.req_rw := mem_req_arb.io.out.bits.rw - io.mem.req_tag := mem_req_arb.io.out.bits.tag - io.mem.req_addr := mem_req_arb.io.out.bits.addr - io.mem.req_wdata := MuxLookup(mem_req_arb.io.out.bits.data_idx, - Bits(0, width = MEM_DATA_BITS), - (0 until NTILES).map( j => - UFix(j) -> Mux(mem_req_arb.io.out.bits.is_probe_rep, - io.tiles(j).probe_rep_data.bits.data, - io.tiles(j).xact_init_data.bits.data))) + //mem_req_arb.io.out.ready := io.mem.req_cmd.ready || io.mem.req_data.ready + io.mem.req_cmd <> mem_req_arb.io.out.bits.req_cmd + io.mem.req_data <> mem_req_arb.io.out.bits.req_data + //io.mem.req_wdata := MuxLookup(mem_req_arb.io.out.bits.data_idx, + // Bits(0, width = MEM_DATA_BITS), + // (0 until NTILES).map( j => + // UFix(j) -> Mux(mem_req_arb.io.out.bits.is_probe_rep, + // io.tiles(j).probe_rep_data.bits.data, + // io.tiles(j).xact_init_data.bits.data))) // Handle probe replies, which may or may not have data for( j <- 0 until NTILES ) { val p_rep = io.tiles(j).probe_rep val p_rep_data = io.tiles(j).probe_rep_data val idx = p_rep.bits.global_xact_id - p_rep_has_data_arr.write(idx, p_rep.valid && p_rep.bits.has_data && p_rep_data.valid) - p_rep_data_idx_arr.write(idx, UFix(j)) p_rep.ready := foldR(trackerList.map(_.io.pop_p_rep(j)))(_ || _) p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) } for( i <- 0 until NGLOBAL_XACTS ) { + trackerList(i).io.p_rep_data := MuxLookup(trackerList(i).io.p_rep_tile_id, Bits(0), (0 until NTILES).map { j => UFix(j) -> io.tiles(j).probe_rep_data }) val flags = Bits(width = NTILES) for( j <- 0 until NTILES) { val p_rep = io.tiles(j).probe_rep - flags(j) := p_rep.valid && !p_rep.bits.has_data && (p_rep.bits.global_xact_id === UFix(i)) + flags(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) } p_rep_cnt_dec_arr.write(UFix(i), flags) } @@ -448,6 +479,8 @@ class CoherenceHubNoDir extends CoherenceHub { trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready trackerList(i).io.alloc_req.bits := init_arb.io.out.bits trackerList(i).io.alloc_req.valid := init_arb.io.out.valid + + trackerList(i).io.x_init_data := MuxLookup(trackerList(i).io.init_tile_id, Bits(0), (0 until NTILES).map { j => UFix(j) -> io.tiles(j).xact_init_data }) } for( j <- 0 until NTILES ) { From eec369a1c74d019443181b96afb21acfb4eb8e36 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 28 Feb 2012 18:59:15 -0800 Subject: [PATCH 022/374] separate memory request command and data also, merge some VLSI/C++ test harness functionality --- uncore/coherence.scala | 6 ------ 1 file changed, 6 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 11c764ec..75c430a3 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -424,12 +424,6 @@ class CoherenceHubNoDir extends CoherenceHub { //mem_req_arb.io.out.ready := io.mem.req_cmd.ready || io.mem.req_data.ready io.mem.req_cmd <> mem_req_arb.io.out.bits.req_cmd io.mem.req_data <> mem_req_arb.io.out.bits.req_data - //io.mem.req_wdata := MuxLookup(mem_req_arb.io.out.bits.data_idx, - // Bits(0, width = MEM_DATA_BITS), - // (0 until NTILES).map( j => - // UFix(j) -> Mux(mem_req_arb.io.out.bits.is_probe_rep, - // io.tiles(j).probe_rep_data.bits.data, - // io.tiles(j).xact_init_data.bits.data))) // Handle probe replies, which may or may not have data for( j <- 0 until NTILES ) { From 6304aa992f92a131c5880d86c0cb659c2aa9a57f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 29 Feb 2012 00:44:03 -0800 Subject: [PATCH 023/374] Fixed race between read resps/reps and write req/reps in null hub --- uncore/coherence.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 75c430a3..d7443d71 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -329,19 +329,19 @@ class CoherenceHubNull extends Component { val x_init = io.tile.xact_init val is_write = x_init.bits.t_type === X_WRITE_UNCACHED - x_init.ready := io.mem.req_cmd.ready - io.mem.req_cmd.valid := x_init.valid + x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp + io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write io.mem.req_cmd.bits.tag := x_init.bits.tile_xact_id io.mem.req_cmd.bits.addr := x_init.bits.address io.mem.req_data <> io.tile.xact_init_data val x_rep = io.tile.xact_rep - x_rep.bits.t_type := X_READ_EXCLUSIVE + x_rep.bits.t_type := Mux(is_write, X_WRITE_UNCACHED, X_READ_EXCLUSIVE) x_rep.bits.tile_xact_id := Mux(is_write, x_init.bits.tile_xact_id, io.mem.resp.tag) x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.data - x_rep.valid := io.mem.resp.valid + x_rep.valid := io.mem.resp.valid || is_write } From ef94f13087ab0758cae3958965d746c496e705ea Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 29 Feb 2012 02:59:27 -0800 Subject: [PATCH 024/374] Broadcast hub nears completion. Still does not handle generation/arbitration for decoupled mem reqs. --- uncore/coherence.scala | 107 +++++++++++++++++++++++++---------------- 1 file changed, 66 insertions(+), 41 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index d7443d71..75796a9d 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -33,9 +33,9 @@ class HubMemReq extends Bundle { val req_data = (new ioDecoupled) { new MemData() } } -class HubProbeRep extends Bundle { - val reply = (new ioDecoupled) { new ProbeReply } - val data_idx = Bits(width = log2up(NTILES)) +class TrackerProbeData extends Bundle { + val valid = Bool() + val data_tile_id = Bits(width = log2up(NTILES)) } class TrackerAllocReq extends Bundle { @@ -180,10 +180,9 @@ trait FourStateCoherence extends CoherencePolicy { class XactTracker(id: Int) extends Component with CoherencePolicy { val io = new Bundle { val alloc_req = (new ioDecoupled) { new TrackerAllocReq() } - val probe_rep = (new ioDecoupled) { new HubProbeRep() } + val probe_data = (new TrackerProbeData).asInput val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) - val p_rep_has_data = Bool(INPUT) val p_rep_cnt_dec = Bits(NTILES, INPUT) val p_req_cnt_inc = Bits(NTILES, INPUT) val p_rep_data = (new ioDecoupled) { new ProbeReplyData() } @@ -207,13 +206,21 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { } def sendProbeReqType(t_type: UFix, global_state: UFix): UFix = { - MuxCase(P_COPY, Array((t_type === X_READ_SHARED) -> P_DOWNGRADE, + MuxCase(P_COPY, Array((t_type === X_READ_SHARED) -> P_DOWNGRADE, (t_type === X_READ_EXCLUSIVE) -> P_INVALIDATE, - (t_type === X_READ_UNCACHED) -> P_COPY, + (t_type === X_READ_UNCACHED) -> P_COPY, (t_type === X_WRITE_UNCACHED) -> P_INVALIDATE)) } - val s_idle :: s_mem_r :: s_mem_w :: s_mem_wr :: s_probe :: s_busy :: Nil = Enum(6){ UFix() } + def needsMemRead(t_type: UFix, global_state: UFix): Bool = { + (t_type != X_WRITE_UNCACHED) + } + + def needsAckRep(t_type: UFix, global_state: UFix): Bool = { + (t_type === X_WRITE_UNCACHED) + } + + val s_idle :: s_mem :: s_probe :: s_busy :: Nil = Enum(4){ UFix() } val state = Reg(resetVal = s_idle) val addr_ = Reg{ Bits() } val t_type_ = Reg{ Bits() } @@ -223,9 +230,10 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { val mem_count = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) val p_rep_count = Reg(resetVal = UFix(0, width = log2up(NTILES))) val p_req_flags = Reg(resetVal = UFix(0, width = NTILES)) - val p_rep_data_idx_ = Reg{ Bits() } - val x_init_data_needs_wb = Reg{ Bool() } - val p_rep_data_needs_wb = Reg{ Bool() } + val p_rep_tile_id_ = Reg{ Bits() } + val x_needs_read = Reg{ Bool() } + val x_init_data_needs_write = Reg{ Bool() } + val p_rep_data_needs_write = Reg{ Bool() } io.busy := state != s_idle io.addr := addr_ @@ -235,15 +243,17 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { io.t_type := t_type_ io.mem_req.valid := Bool(false) - io.mem_req.bits.req_cmd.bits.rw := state === s_mem_w || state === s_mem_wr + io.mem_req.bits.req_cmd.valid := Bool(false) + io.mem_req.bits.req_cmd.bits.rw := Bool(false) io.mem_req.bits.req_cmd.bits.addr := addr_ io.mem_req.bits.req_cmd.bits.tag := UFix(id) + io.mem_req.bits.req_data.valid := Bool(false) + io.mem_req.bits.req_data.bits.data := UFix(0) // := io.mem.ready //sent mem req io.probe_req.valid := Bool(false) io.probe_req.bits.p_type := sendProbeReqType(t_type_, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) io.probe_req.bits.address := addr_ - // := io.probe_req.ready //got through arbiter ---- p_rep_dec_arr io.push_p_req := Bits(0, width = NTILES) io.pop_p_rep := Bits(0, width = NTILES) io.pop_p_rep_data := Bits(0, width = NTILES) @@ -258,32 +268,14 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { t_type_ := io.alloc_req.bits.xact_init.t_type init_tile_id_ := io.alloc_req.bits.init_tile_id tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id - x_init_data_needs_wb := io.alloc_req.bits.xact_init.has_data + x_init_data_needs_write := io.alloc_req.bits.xact_init.has_data + x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) p_rep_count := UFix(NTILES) p_req_flags := ~Bits(0, width = NTILES) state := s_probe io.pop_x_init := Bool(true) } } - is(s_mem_r) { - io.mem_req.valid := Bool(true) - when(io.mem_req.ready) { state := s_busy } - } - is(s_mem_w) { - io.mem_req.valid := Bool(true) - when(io.mem_req.ready) { state := s_busy } - } - is(s_mem_wr) { - when(io.probe_rep.bits.reply.bits.has_data) { - //io.pop_p_rep(p_rep_data_idx) := io.mem_req_rdy - //io.pop_p_rep_data(p_rep_data_idx) := io.mem_req_rdy //TODO - } . otherwise { - //io.pop_x_init := io.mem_req_rdy - //io.pop_x_init_data := io.mem_req_rdy - } - io.mem_req.valid := Bool(true) - when(io.mem_req.ready) { state := s_mem_r } - } is(s_probe) { when(p_req_flags.orR) { io.push_p_req := p_req_flags @@ -292,22 +284,55 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { when(io.p_req_cnt_inc.orR) { p_req_flags := p_req_flags & ~io.p_req_cnt_inc // unflag sent reqs } - val p_rep_has_data = Bool(INPUT) - val p_rep_data_idx = Bits(log2up(NTILES), INPUT) - val p_rep_cnt_dec = Bits(NTILES, INPUT) when(io.p_rep_cnt_dec.orR) { val p_rep_count_next = p_rep_count - PopCount(io.p_rep_cnt_dec) p_rep_count := p_rep_count_next when(p_rep_count_next === UFix(0)) { - state := s_busy //TODO: XXXXXXXXXX + state := s_mem } } - when(p_rep_has_data) { - p_rep_data_needs_wb := Bool(true) - p_rep_data_idx_ := p_rep_data_idx + when(io.probe_data.valid) { + p_rep_data_needs_write := Bool(true) + p_rep_tile_id_ := io.p_rep_tile_id } } - is(s_busy) { + is(s_mem) { + when(x_init_data_needs_write) { + //io.mem_req.valid := //?TODO ??? || io.x_init_data.valid + //io.mem_req.bits.req_cmd.valid := // TODO ??? + io.mem_req.bits.req_cmd.bits.rw := Bool(true) + io.mem_req.bits.req_data <> io.x_init_data + when(io.mem_req.ready && io.mem_req.bits.req_cmd.ready) { + //TODO + } + when(io.mem_req.ready && io.mem_req.bits.req_data.ready) { + io.pop_x_init_data := Bool(true) + //TODO: count with mem_count somehow + } + } . elsewhen (p_rep_data_needs_write) { + //io.mem_req.valid := //TODO ??? || io.p_rep_data.valid + //io.mem_req.bits.req_cmd.valid := //TODO ??? + io.mem_req.bits.req_cmd.bits.rw := Bool(true) + io.mem_req.bits.req_data <> io.p_rep_data + when(io.mem_req.ready && io.mem_req.bits.req_cmd.ready) { + //TODO + } + when(io.mem_req.ready && io.mem_req.bits.req_data.ready) { + io.pop_p_rep_data := Bool(true) + //TODO: count with mem_count somehow + } + } . elsewhen (x_needs_read) { + io.mem_req.valid := Bool(true) + io.mem_req.bits.req_cmd.valid := Bool(true) + when(io.mem_req.ready && io.mem_req.bits.req_cmd.ready) { + x_needs_read := Bool(false) + } + } . otherwise { + io.send_x_rep_ack := needsAckRep(t_type_, UFix(0)) + state := s_busy + } + } + is(s_busy) { // Nothing left to do but wait for transaction to complete when (io.xact_finish) { state := s_idle } From 8b519e7ea8d578f66597ccd148aa5ec1fa7ee331 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 29 Feb 2012 03:08:04 -0800 Subject: [PATCH 025/374] replace tile memory interface with ioTileLink work in progress towards coherent HTIF. for now, requests are incoherently passed through a null coherence hub. --- uncore/coherence.scala | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 75796a9d..9fbd69e9 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -10,22 +10,21 @@ class MemData extends Bundle { class MemReqCmd() extends Bundle { val rw = Bool() - val addr = UFix(PADDR_BITS - OFFSET_BITS) - val tag = Bits(MEM_TAG_BITS) + val addr = UFix(width = PADDR_BITS - OFFSET_BITS) + val tag = Bits(width = MEM_TAG_BITS) } class MemResp () extends Bundle { - val tag = Bits(MEM_TAG_BITS) + val tag = Bits(width = MEM_TAG_BITS) val data = Bits(width = MEM_DATA_BITS) - val valid = Bool() } -class ioMemHub() extends Bundle +class ioMem() extends Bundle { val req_cmd = (new ioDecoupled) { new MemReqCmd() }.flip val req_data = (new ioDecoupled) { new MemData() }.flip - val resp = new MemResp() + val resp = (new ioValid) { new MemResp() } } class HubMemReq extends Bundle { @@ -49,7 +48,7 @@ class TransactionInit extends Bundle { val t_type = Bits(width = TTYPE_BITS) val has_data = Bool() val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) - val address = Bits(width = PADDR_BITS) + val address = UFix(width = PADDR_BITS) } class TransactionInitData extends MemData @@ -348,8 +347,8 @@ abstract class CoherenceHub extends Component with CoherencePolicy class CoherenceHubNull extends Component { val io = new Bundle { - val tile = new ioTileLink() - val mem = new ioMemHub() + val tile = new ioTileLink().flip + val mem = new ioMem } val x_init = io.tile.xact_init @@ -362,11 +361,11 @@ class CoherenceHubNull extends Component { io.mem.req_data <> io.tile.xact_init_data val x_rep = io.tile.xact_rep - x_rep.bits.t_type := Mux(is_write, X_WRITE_UNCACHED, X_READ_EXCLUSIVE) - x_rep.bits.tile_xact_id := Mux(is_write, x_init.bits.tile_xact_id, io.mem.resp.tag) + x_rep.bits.t_type := Mux(io.mem.resp.valid, X_READ_EXCLUSIVE, X_WRITE_UNCACHED) + x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) x_rep.bits.global_xact_id := UFix(0) // don't care - x_rep.bits.data := io.mem.resp.data - x_rep.valid := io.mem.resp.valid || is_write + x_rep.bits.data := io.mem.resp.bits.data + x_rep.valid := io.mem.resp.valid || x_init.valid && is_write } @@ -388,7 +387,7 @@ class CoherenceHubNoDir extends CoherenceHub { val io = new Bundle { val tiles = Vec(NTILES) { new ioTileLink() } - val mem = new ioMemHub + val mem = new ioMem } val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) @@ -427,12 +426,12 @@ class CoherenceHubNoDir extends CoherenceHub { // Reply to initial requestor // Forward memory responses from mem to tile - val idx = io.mem.resp.tag + val idx = io.mem.resp.bits.tag for( j <- 0 until NTILES ) { io.tiles(j).xact_rep.bits.t_type := getTransactionReplyType(t_type_arr.read(idx), sh_count_arr.read(idx)) io.tiles(j).xact_rep.bits.tile_xact_id := tile_xact_id_arr.read(idx) io.tiles(j).xact_rep.bits.global_xact_id := idx - io.tiles(j).xact_rep.bits.data := io.mem.resp.data + io.tiles(j).xact_rep.bits.data := io.mem.resp.bits.data io.tiles(j).xact_rep.valid := (UFix(j) === init_tile_id_arr.read(idx)) && (io.mem.resp.valid || send_x_rep_ack_arr.read(idx)) } // If there were a ready signal due to e.g. intervening network use: From 7e6d990bad5060fbed0dead78e902ba9ce3be04f Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Wed, 29 Feb 2012 17:12:02 -0800 Subject: [PATCH 026/374] Merge branch 'master' of github.com:ucb-bar/riscv-rocket From 4f00bcc760ebdd1a41451857f85dc527dbaddf97 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Wed, 29 Feb 2012 17:12:02 -0800 Subject: [PATCH 027/374] Merge branch 'master' of github.com:ucb-bar/riscv-rocket From 940027a888eaf9e9409186a2939ffb3ec5fadafd Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 29 Feb 2012 17:58:15 -0800 Subject: [PATCH 028/374] Finished broadcast hub with split mem req types. Untested. --- uncore/coherence.scala | 119 ++++++++++++++++++++++------------------- 1 file changed, 64 insertions(+), 55 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 9fbd69e9..f63e633d 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -28,8 +28,7 @@ class ioMem() extends Bundle } class HubMemReq extends Bundle { - val req_cmd = (new ioDecoupled) { new MemReqCmd() } - val req_data = (new ioDecoupled) { new MemData() } + val lock = Bool() } class TrackerProbeData extends Bundle { @@ -187,7 +186,9 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { val p_rep_data = (new ioDecoupled) { new ProbeReplyData() } val x_init_data = (new ioDecoupled) { new TransactionInitData() } - val mem_req = (new ioDecoupled) { new HubMemReq() }.flip + val mem_req_cmd = (new ioDecoupled) { new MemReqCmd() } + val mem_req_data = (new ioDecoupled) { new MemData() } + val mem_req_lock = Bool(OUTPUT) val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip val busy = Bool(OUTPUT) val addr = Bits(PADDR_BITS, OUTPUT) @@ -225,14 +226,40 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { val t_type_ = Reg{ Bits() } val init_tile_id_ = Reg{ Bits() } val tile_xact_id_ = Reg{ Bits() } - val probe_done = Reg{ Bits() } - val mem_count = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) val p_rep_count = Reg(resetVal = UFix(0, width = log2up(NTILES))) val p_req_flags = Reg(resetVal = UFix(0, width = NTILES)) val p_rep_tile_id_ = Reg{ Bits() } - val x_needs_read = Reg{ Bool() } - val x_init_data_needs_write = Reg{ Bool() } - val p_rep_data_needs_write = Reg{ Bool() } + val x_needs_read = Reg(resetVal = Bool(false)) + val x_init_data_needs_write = Reg(resetVal = Bool(false)) + val p_rep_data_needs_write = Reg(resetVal = Bool(false)) + val mem_cmd_sent = Reg(resetVal = Bool(false)) + val mem_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val mem_cnt_next = mem_cnt + UFix(1) + + def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioDecoupled[MemData], trigger: Bool, pop: Bool) { + req_cmd.valid := mem_cmd_sent + req_cmd.bits.rw := Bool(true) + req_data <> data + lock := Bool(true) + when(req_cmd.ready && req_cmd.valid) { + mem_cmd_sent := Bool(false) + } + when(req_data.ready && req_data.valid) { + pop := Bool(true) + mem_cnt := mem_cnt_next + } + when(mem_cnt === ~UFix(0)) { + trigger := Bool(false) + } + } + + def doMemReqRead(req_cmd: ioDecoupled[MemReqCmd], trigger: Bool) { + req_cmd.valid := Bool(true) + req_cmd.bits.rw := Bool(false) + when(req_cmd.ready ) { + trigger := Bool(false) + } + } io.busy := state != s_idle io.addr := addr_ @@ -241,14 +268,13 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { io.sharer_count := UFix(NTILES) // TODO: Broadcast only io.t_type := t_type_ - io.mem_req.valid := Bool(false) - io.mem_req.bits.req_cmd.valid := Bool(false) - io.mem_req.bits.req_cmd.bits.rw := Bool(false) - io.mem_req.bits.req_cmd.bits.addr := addr_ - io.mem_req.bits.req_cmd.bits.tag := UFix(id) - io.mem_req.bits.req_data.valid := Bool(false) - io.mem_req.bits.req_data.bits.data := UFix(0) - // := io.mem.ready //sent mem req + io.mem_req_cmd.valid := Bool(false) + io.mem_req_cmd.bits.rw := Bool(false) + io.mem_req_cmd.bits.addr := addr_ + io.mem_req_cmd.bits.tag := UFix(id) + io.mem_req_data.valid := Bool(false) + io.mem_req_data.bits.data := UFix(0) + io.mem_req_lock := Bool(false) io.probe_req.valid := Bool(false) io.probe_req.bits.p_type := sendProbeReqType(t_type_, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) @@ -269,9 +295,11 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := io.alloc_req.bits.xact_init.has_data x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) - p_rep_count := UFix(NTILES) - p_req_flags := ~Bits(0, width = NTILES) - state := s_probe + p_rep_count := UFix(NTILES-1) + p_req_flags := ~( UFix(1) << io.alloc_req.bits.init_tile_id ) + state := Mux(p_req_flags.orR, s_probe, s_mem) + mem_cnt := UFix(0) + mem_cmd_sent := Bool(false) io.pop_x_init := Bool(true) } } @@ -285,8 +313,11 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { } when(io.p_rep_cnt_dec.orR) { val p_rep_count_next = p_rep_count - PopCount(io.p_rep_cnt_dec) + io.pop_p_rep := io.p_rep_cnt_dec p_rep_count := p_rep_count_next when(p_rep_count_next === UFix(0)) { + mem_cnt := UFix(0) + mem_cmd_sent := Bool(false) state := s_mem } } @@ -296,36 +327,12 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { } } is(s_mem) { - when(x_init_data_needs_write) { - //io.mem_req.valid := //?TODO ??? || io.x_init_data.valid - //io.mem_req.bits.req_cmd.valid := // TODO ??? - io.mem_req.bits.req_cmd.bits.rw := Bool(true) - io.mem_req.bits.req_data <> io.x_init_data - when(io.mem_req.ready && io.mem_req.bits.req_cmd.ready) { - //TODO - } - when(io.mem_req.ready && io.mem_req.bits.req_data.ready) { - io.pop_x_init_data := Bool(true) - //TODO: count with mem_count somehow - } - } . elsewhen (p_rep_data_needs_write) { - //io.mem_req.valid := //TODO ??? || io.p_rep_data.valid - //io.mem_req.bits.req_cmd.valid := //TODO ??? - io.mem_req.bits.req_cmd.bits.rw := Bool(true) - io.mem_req.bits.req_data <> io.p_rep_data - when(io.mem_req.ready && io.mem_req.bits.req_cmd.ready) { - //TODO - } - when(io.mem_req.ready && io.mem_req.bits.req_data.ready) { - io.pop_p_rep_data := Bool(true) - //TODO: count with mem_count somehow - } + when (p_rep_data_needs_write) { + doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, io.p_rep_data, p_rep_data_needs_write, io.pop_p_rep_data) + } . elsewhen(x_init_data_needs_write) { + doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, io.x_init_data, x_init_data_needs_write, io.pop_x_init_data) } . elsewhen (x_needs_read) { - io.mem_req.valid := Bool(true) - io.mem_req.bits.req_cmd.valid := Bool(true) - when(io.mem_req.ready && io.mem_req.bits.req_cmd.ready) { - x_needs_read := Bool(false) - } + doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { io.send_x_rep_ack := needsAckRep(t_type_, UFix(0)) state := s_busy @@ -369,7 +376,7 @@ class CoherenceHubNull extends Component { } -class CoherenceHubNoDir extends CoherenceHub { +class CoherenceHubBroadcast extends CoherenceHub { def coherenceConflict(addr1: Bits, addr2: Bits): Bool = { addr1(PADDR_BITS-1, OFFSET_BITS) === addr2(PADDR_BITS-1, OFFSET_BITS) @@ -440,14 +447,16 @@ class CoherenceHubNoDir extends CoherenceHub { // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests // and once we have picked a request, get the right write data - - val mem_req_arb = (new Arbiter(NGLOBAL_XACTS)) { new HubMemReq() } + val mem_req_cmd_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemReqCmd() } + val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemData() } for( i <- 0 until NGLOBAL_XACTS ) { - mem_req_arb.io.in(i) <> trackerList(i).io.mem_req + mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd + mem_req_cmd_arb.io.lock(i) <> trackerList(i).io.mem_req_lock + mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data + mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock } - //mem_req_arb.io.out.ready := io.mem.req_cmd.ready || io.mem.req_data.ready - io.mem.req_cmd <> mem_req_arb.io.out.bits.req_cmd - io.mem.req_data <> mem_req_arb.io.out.bits.req_data + io.mem.req_cmd <> mem_req_cmd_arb.io.out + io.mem.req_data <> mem_req_data_arb.io.out // Handle probe replies, which may or may not have data for( j <- 0 until NTILES ) { From 4e33b68beaa90dcc3646c3c305123286427e369b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 1 Mar 2012 01:19:09 -0800 Subject: [PATCH 029/374] Unified hub ios. Fixed some hub elaboration errors. --- uncore/coherence.scala | 52 +++++++++++++++++------------------------- 1 file changed, 21 insertions(+), 31 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index f63e633d..0ded6f22 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -222,12 +222,12 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { val s_idle :: s_mem :: s_probe :: s_busy :: Nil = Enum(4){ UFix() } val state = Reg(resetVal = s_idle) - val addr_ = Reg{ Bits() } + val addr_ = Reg{ UFix() } val t_type_ = Reg{ Bits() } val init_tile_id_ = Reg{ Bits() } val tile_xact_id_ = Reg{ Bits() } val p_rep_count = Reg(resetVal = UFix(0, width = log2up(NTILES))) - val p_req_flags = Reg(resetVal = UFix(0, width = NTILES)) + val p_req_flags = Reg(resetVal = Bits(0, width = NTILES)) val p_rep_tile_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) val x_init_data_needs_write = Reg(resetVal = Bool(false)) @@ -350,24 +350,25 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { // P_rep and x_init must be popped on same cycle of receipt } -abstract class CoherenceHub extends Component with CoherencePolicy - -class CoherenceHubNull extends Component { +abstract class CoherenceHub extends Component with CoherencePolicy { val io = new Bundle { - val tile = new ioTileLink().flip + val tiles = Vec(NTILES) { new ioTileLink() }.flip val mem = new ioMem } +} - val x_init = io.tile.xact_init +class CoherenceHubNull extends CoherenceHub { + + val x_init = io.tiles(0).xact_init val is_write = x_init.bits.t_type === X_WRITE_UNCACHED x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write io.mem.req_cmd.bits.tag := x_init.bits.tile_xact_id io.mem.req_cmd.bits.addr := x_init.bits.address - io.mem.req_data <> io.tile.xact_init_data + io.mem.req_data <> io.tiles(0).xact_init_data - val x_rep = io.tile.xact_rep + val x_rep = io.tiles(0).xact_rep x_rep.bits.t_type := Mux(io.mem.resp.valid, X_READ_EXCLUSIVE, X_WRITE_UNCACHED) x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) x_rep.bits.global_xact_id := UFix(0) // don't care @@ -392,13 +393,9 @@ class CoherenceHubBroadcast extends CoherenceHub { ret } - val io = new Bundle { - val tiles = Vec(NTILES) { new ioTileLink() } - val mem = new ioMem - } - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) +/* val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } @@ -467,13 +464,12 @@ class CoherenceHubBroadcast extends CoherenceHub { p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) } for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.p_rep_data := MuxLookup(trackerList(i).io.p_rep_tile_id, Bits(0), (0 until NTILES).map { j => UFix(j) -> io.tiles(j).probe_rep_data }) - val flags = Bits(width = NTILES) + trackerList(i).io.p_rep_data <> io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data for( j <- 0 until NTILES) { val p_rep = io.tiles(j).probe_rep - flags(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) + val dec = p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) + p_rep_cnt_dec_arr(UFix(i)) := p_rep_cnt_dec_arr(UFix(i)).bitSet(UFix(j), dec) } - p_rep_cnt_dec_arr.write(UFix(i), flags) } // Nack conflicting transaction init attempts @@ -484,8 +480,8 @@ class CoherenceHubBroadcast extends CoherenceHub { val conflicts = Bits(width = NGLOBAL_XACTS) for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(i) := t.busy(i) && coherenceConflict(t.addr, x_init.bits.address) && - !(x_init.bits.has_data && (UFix(j) === t.init_tile_id)) + conflicts(UFix(i), t.busy(i) && coherenceConflict(t.addr, x_init.bits.address) && + !(x_init.bits.has_data && (UFix(j) === t.init_tile_id))) // Don't abort writebacks stalled on mem. // TODO: This assumes overlapped writeback init reqs to // the same addr will never be issued; is this ok? @@ -493,7 +489,7 @@ class CoherenceHubBroadcast extends CoherenceHub { x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id val want_to_abort = conflicts.orR || busy_arr.toBits.andR x_abort.valid := want_to_abort && x_init.valid - aborting(j) := want_to_abort && x_abort.ready + aborting.bitSet(UFix(j), want_to_abort && x_abort.ready) } // Handle transaction initiation requests @@ -504,10 +500,10 @@ class CoherenceHubBroadcast extends CoherenceHub { for( i <- 0 until NGLOBAL_XACTS ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready - trackerList(i).io.alloc_req.bits := init_arb.io.out.bits + trackerList(i).io.alloc_req.bits <> init_arb.io.out.bits trackerList(i).io.alloc_req.valid := init_arb.io.out.valid - trackerList(i).io.x_init_data := MuxLookup(trackerList(i).io.init_tile_id, Bits(0), (0 until NTILES).map { j => UFix(j) -> io.tiles(j).xact_init_data }) + trackerList(i).io.x_init_data <> io.tiles(trackerList(i).io.init_tile_id).xact_init_data } for( j <- 0 until NTILES ) { @@ -533,15 +529,9 @@ class CoherenceHubBroadcast extends CoherenceHub { val t = trackerList(i).io p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits p_req_arb_arr(j).io.in(i).valid := t.probe_req.valid && t.push_p_req(j) + p_rep_cnt_dec_arr(i) = p_rep_cnt_dec_arr(i).bitSet(UFix(j), p_req_arb_arr(j).io.in(i).ready) } p_req_arb_arr(j).io.out <> io.tiles(j).probe_req } - for( i <- 0 until NGLOBAL_XACTS ) { - val flags = Bits(width = NTILES) - for( j <- 0 until NTILES ) { - flags(j) := p_req_arb_arr(j).io.in(i).ready - } - p_rep_cnt_dec_arr.write(UFix(i), flags) - } - +*/ } From 2152b0283d0a3438b644210166d2122ba64c2580 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 1 Mar 2012 17:03:56 -0800 Subject: [PATCH 030/374] Made xact_rep an ioValid, removed has_data member --- uncore/coherence.scala | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 0ded6f22..12c1e4a9 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -14,10 +14,9 @@ class MemReqCmd() extends Bundle val tag = Bits(width = MEM_TAG_BITS) } -class MemResp () extends Bundle +class MemResp () extends MemData { val tag = Bits(width = MEM_TAG_BITS) - val data = Bits(width = MEM_DATA_BITS) } class ioMem() extends Bundle @@ -72,7 +71,6 @@ class ProbeReplyData extends MemData class TransactionReply extends MemData { val t_type = Bits(width = TTYPE_BITS) - val has_data = Bool() val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } @@ -88,7 +86,7 @@ class ioTileLink extends Bundle { val probe_req = (new ioDecoupled) { new ProbeRequest() } val probe_rep = (new ioDecoupled) { new ProbeReply() }.flip val probe_rep_data = (new ioDecoupled) { new ProbeReplyData() }.flip - val xact_rep = (new ioDecoupled) { new TransactionReply() } + val xact_rep = (new ioValid) { new TransactionReply() } val xact_finish = (new ioDecoupled) { new TransactionFinish() }.flip } @@ -173,6 +171,10 @@ trait FourStateCoherence extends CoherencePolicy { } state.toBits } + + def replyTypeHasData (reply: TransactionReply): Bool = { + (reply.t_type != X_WRITE_UNCACHED) + } } class XactTracker(id: Int) extends Component with CoherencePolicy { From de73eef4091d41ed41ed824fe65ab19927dfd623 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 1 Mar 2012 18:23:46 -0800 Subject: [PATCH 031/374] Fixed elaboration errors in LockingArbiter and BoradcastHub. Fixed ioDecoupled direction error in XactTracker --- uncore/coherence.scala | 130 ++++++++++++++++++++++++----------------- 1 file changed, 76 insertions(+), 54 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 12c1e4a9..5f70d81f 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -163,13 +163,12 @@ trait FourStateCoherence extends CoherencePolicy { def needsSecondaryXact (cmd: Bits, outstanding: TransactionInit): Bool - def getMetaUpdateOnProbe (incoming: ProbeRequest): Bits = { - val state = UFix(0) - switch(incoming.p_type) { - is(probeInvalidate) { state := tileInvalid } - is(probeDowngrade) { state := tileShared } - } - state.toBits + def newStateOnProbe (incoming: ProbeRequest, state: UFix): Bits = { + MuxLookup(incoming.p_type, state, Array( + probeInvalidate -> tileInvalid, + probeDowngrade -> tileShared, + probeCopy -> state + )) } def replyTypeHasData (reply: TransactionReply): Bool = { @@ -187,9 +186,10 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { val p_req_cnt_inc = Bits(NTILES, INPUT) val p_rep_data = (new ioDecoupled) { new ProbeReplyData() } val x_init_data = (new ioDecoupled) { new TransactionInitData() } + val sent_x_rep_ack = Bool(INPUT) - val mem_req_cmd = (new ioDecoupled) { new MemReqCmd() } - val mem_req_data = (new ioDecoupled) { new MemData() } + val mem_req_cmd = (new ioDecoupled) { new MemReqCmd() }.flip + val mem_req_data = (new ioDecoupled) { new MemData() }.flip val mem_req_lock = Bool(OUTPUT) val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip val busy = Bool(OUTPUT) @@ -222,7 +222,7 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { (t_type === X_WRITE_UNCACHED) } - val s_idle :: s_mem :: s_probe :: s_busy :: Nil = Enum(4){ UFix() } + val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) val addr_ = Reg{ UFix() } val t_type_ = Reg{ Bits() } @@ -241,7 +241,10 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioDecoupled[MemData], trigger: Bool, pop: Bool) { req_cmd.valid := mem_cmd_sent req_cmd.bits.rw := Bool(true) - req_data <> data + //TODO: why does req_data <> data segfault? + req_data.valid := data.valid + req_data.bits.data := data.bits.data + data.ready := req_data.ready lock := Bool(true) when(req_cmd.ready && req_cmd.valid) { mem_cmd_sent := Bool(false) @@ -270,7 +273,7 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { io.sharer_count := UFix(NTILES) // TODO: Broadcast only io.t_type := t_type_ - io.mem_req_cmd.valid := Bool(false) + io.mem_req_cmd.valid := Bool(false) io.mem_req_cmd.bits.rw := Bool(false) io.mem_req_cmd.bits.addr := addr_ io.mem_req_cmd.bits.tag := UFix(id) @@ -279,7 +282,7 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { io.mem_req_lock := Bool(false) io.probe_req.valid := Bool(false) io.probe_req.bits.p_type := sendProbeReqType(t_type_, UFix(0)) - io.probe_req.bits.global_xact_id := UFix(id) + io.probe_req.bits.global_xact_id := UFix(id) io.probe_req.bits.address := addr_ io.push_p_req := Bits(0, width = NTILES) io.pop_p_rep := Bits(0, width = NTILES) @@ -287,6 +290,8 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { io.pop_x_init := Bool(false) io.pop_x_init_data := Bool(false) io.send_x_rep_ack := Bool(false) + io.x_init_data.ready := Bool(false) // don't care + io.p_rep_data.ready := Bool(false) // don't care switch (state) { is(s_idle) { @@ -336,20 +341,19 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { - io.send_x_rep_ack := needsAckRep(t_type_, UFix(0)) - state := s_busy + state := Mux(needsAckRep(t_type_, UFix(0)), s_ack, s_busy) } } + is(s_ack) { + io.send_x_rep_ack := Bool(true) + when(io.sent_x_rep_ack) { state := s_busy } + } is(s_busy) { // Nothing left to do but wait for transaction to complete when (io.xact_finish) { state := s_idle } } } - - //TODO: Decrement the probe count when final data piece is written - // Connent io.mem.ready sig to correct pop* outputs - // P_rep and x_init must be popped on same cycle of receipt } abstract class CoherenceHub extends Component with CoherencePolicy { @@ -385,19 +389,16 @@ class CoherenceHubBroadcast extends CoherenceHub { addr1(PADDR_BITS-1, OFFSET_BITS) === addr2(PADDR_BITS-1, OFFSET_BITS) } def getTransactionReplyType(t_type: UFix, count: UFix): Bits = { - val ret = Wire() { Bits(width = TTYPE_BITS) } - switch (t_type) { - is(X_READ_SHARED) { ret := Mux(count > UFix(0), X_READ_SHARED, X_READ_EXCLUSIVE) } - is(X_READ_EXCLUSIVE) { ret := X_READ_EXCLUSIVE } - is(X_READ_UNCACHED) { ret := X_READ_UNCACHED } - is(X_WRITE_UNCACHED) { ret := X_WRITE_UNCACHED } - } - ret + MuxLookup(t_type, X_READ_UNCACHED, Array( + X_READ_SHARED -> Mux(count > UFix(0), X_READ_SHARED, X_READ_EXCLUSIVE), + X_READ_EXCLUSIVE -> X_READ_EXCLUSIVE, + X_READ_UNCACHED -> X_READ_UNCACHED, + X_WRITE_UNCACHED -> X_WRITE_UNCACHED + )) } val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) -/* val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } @@ -409,40 +410,61 @@ class CoherenceHubBroadcast extends CoherenceHub { val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } val p_rep_cnt_dec_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } val p_req_cnt_inc_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } + val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } for( i <- 0 until NGLOBAL_XACTS) { - busy_arr.write( UFix(i), trackerList(i).io.busy) - addr_arr.write( UFix(i), trackerList(i).io.addr) - init_tile_id_arr.write( UFix(i), trackerList(i).io.init_tile_id) - tile_xact_id_arr.write( UFix(i), trackerList(i).io.tile_xact_id) - t_type_arr.write( UFix(i), trackerList(i).io.t_type) - sh_count_arr.write( UFix(i), trackerList(i).io.sharer_count) - send_x_rep_ack_arr.write(UFix(i), trackerList(i).io.send_x_rep_ack) - trackerList(i).io.xact_finish := do_free_arr.read(UFix(i)) - trackerList(i).io.p_rep_cnt_dec := p_rep_cnt_dec_arr.read(UFix(i)) - trackerList(i).io.p_req_cnt_inc := p_req_cnt_inc_arr.read(UFix(i)) + val t = trackerList(i).io + busy_arr(i) := t.busy + addr_arr(i) := t.addr + init_tile_id_arr(i) := t.init_tile_id + tile_xact_id_arr(i) := t.tile_xact_id + t_type_arr(i) := t.t_type + sh_count_arr(i) := t.sharer_count + send_x_rep_ack_arr(i) := t.send_x_rep_ack + do_free_arr(i) := Bool(false) + p_rep_cnt_dec_arr(i) := Bits(0) + p_req_cnt_inc_arr(i) := Bits(0) + sent_x_rep_ack_arr(i) := Bool(false) + t.xact_finish := do_free_arr(i) + t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i) + t.p_req_cnt_inc := p_req_cnt_inc_arr(i) + t.sent_x_rep_ack := sent_x_rep_ack_arr(i) } // Free finished transactions for( j <- 0 until NTILES ) { val finish = io.tiles(j).xact_finish - do_free_arr.write(finish.bits.global_xact_id, finish.valid) + do_free_arr(finish.bits.global_xact_id) := finish.valid finish.ready := Bool(true) } // Reply to initial requestor - // Forward memory responses from mem to tile - val idx = io.mem.resp.bits.tag + // Forward memory responses from mem to tile or arbitrate to ack + val mem_idx = io.mem.resp.bits.tag + val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits, NGLOBAL_XACTS) for( j <- 0 until NTILES ) { - io.tiles(j).xact_rep.bits.t_type := getTransactionReplyType(t_type_arr.read(idx), sh_count_arr.read(idx)) - io.tiles(j).xact_rep.bits.tile_xact_id := tile_xact_id_arr.read(idx) - io.tiles(j).xact_rep.bits.global_xact_id := idx + val rep = io.tiles(j).xact_rep + rep.bits.t_type := UFix(0) + rep.bits.tile_xact_id := UFix(0) + rep.bits.global_xact_id := UFix(0) + rep.valid := Bool(false) + when(io.mem.resp.valid) { + rep.bits.t_type := getTransactionReplyType(t_type_arr(mem_idx), sh_count_arr(mem_idx)) + rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) + rep.bits.global_xact_id := mem_idx + rep.valid := (UFix(j) === init_tile_id_arr(mem_idx)) + } . otherwise { + rep.bits.t_type := getTransactionReplyType(t_type_arr(ack_idx), sh_count_arr(ack_idx)) + rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) + rep.bits.global_xact_id := ack_idx + rep.valid := (UFix(j) === init_tile_id_arr(ack_idx)) && send_x_rep_ack_arr(ack_idx) + } io.tiles(j).xact_rep.bits.data := io.mem.resp.bits.data - io.tiles(j).xact_rep.valid := (UFix(j) === init_tile_id_arr.read(idx)) && (io.mem.resp.valid || send_x_rep_ack_arr.read(idx)) } + sent_x_rep_ack_arr(ack_idx) := !io.mem.resp.valid && send_x_rep_ack_arr(ack_idx) // If there were a ready signal due to e.g. intervening network use: - //io.mem.resp.ready := io.tiles(init_tile_id_arr.read(idx)).xact_rep.ready - + //io.mem.resp.ready := io.tiles(init_tile_id_arr.read(mem_idx)).xact_rep.ready + // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests // and once we have picked a request, get the right write data @@ -466,7 +488,8 @@ class CoherenceHubBroadcast extends CoherenceHub { p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) } for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.p_rep_data <> io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data + trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid + trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits for( j <- 0 until NTILES) { val p_rep = io.tiles(j).probe_rep val dec = p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) @@ -475,14 +498,14 @@ class CoherenceHubBroadcast extends CoherenceHub { } // Nack conflicting transaction init attempts - val aborting = Wire() { Bits(width = NTILES) } + val aborting = Bits(0, width = NTILES) for( j <- 0 until NTILES ) { val x_init = io.tiles(j).xact_init val x_abort = io.tiles(j).xact_abort val conflicts = Bits(width = NGLOBAL_XACTS) for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(UFix(i), t.busy(i) && coherenceConflict(t.addr, x_init.bits.address) && + conflicts(UFix(i), t.busy && coherenceConflict(t.addr, x_init.bits.address) && !(x_init.bits.has_data && (UFix(j) === t.init_tile_id))) // Don't abort writebacks stalled on mem. // TODO: This assumes overlapped writeback init reqs to @@ -505,9 +528,9 @@ class CoherenceHubBroadcast extends CoherenceHub { trackerList(i).io.alloc_req.bits <> init_arb.io.out.bits trackerList(i).io.alloc_req.valid := init_arb.io.out.valid - trackerList(i).io.x_init_data <> io.tiles(trackerList(i).io.init_tile_id).xact_init_data + trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits + trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid } - for( j <- 0 until NTILES ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data @@ -522,7 +545,6 @@ class CoherenceHubBroadcast extends CoherenceHub { alloc_arb.io.out.ready := init_arb.io.out.valid && !busy_arr.toBits.andR && !foldR(trackerList.map(t => t.io.busy && coherenceConflict(t.io.addr, init_arb.io.out.bits.xact_init.address)))(_||_) - // Handle probe request generation // Must arbitrate for each request port val p_req_arb_arr = List.fill(NTILES)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) @@ -535,5 +557,5 @@ class CoherenceHubBroadcast extends CoherenceHub { } p_req_arb_arr(j).io.out <> io.tiles(j).probe_req } -*/ + } From 4c3f7a36cedb2f49d657ea1bb151694f66f6b339 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 1 Mar 2012 18:49:00 -0800 Subject: [PATCH 032/374] clean up D$ store data unit --- uncore/coherence.scala | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 5f70d81f..2c5c6edf 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -90,8 +90,8 @@ class ioTileLink extends Bundle { val xact_finish = (new ioDecoupled) { new TransactionFinish() }.flip } -trait CoherencePolicy { - def cpuCmdToRW( cmd: Bits): (Bool, Bool) = { +object cpuCmdToRW { + def apply(cmd: Bits): (Bool, Bool) = { val store = (cmd === M_XWR) val load = (cmd === M_XRD) val amo = cmd(3).toBool @@ -101,6 +101,9 @@ trait CoherencePolicy { } } +trait CoherencePolicy { +} + trait ThreeStateIncoherence extends CoherencePolicy { val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } From bef59527d6b155b8e888e1d1d57d9dd171903917 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Thu, 1 Mar 2012 20:48:46 -0800 Subject: [PATCH 033/374] clean up ioDecoupled/ioPipe interface --- uncore/coherence.scala | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 2c5c6edf..97d098bd 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -21,9 +21,9 @@ class MemResp () extends MemData class ioMem() extends Bundle { - val req_cmd = (new ioDecoupled) { new MemReqCmd() }.flip - val req_data = (new ioDecoupled) { new MemData() }.flip - val resp = (new ioValid) { new MemResp() } + val req_cmd = (new ioDecoupled) { new MemReqCmd() } + val req_data = (new ioDecoupled) { new MemData() } + val resp = (new ioPipe) { new MemResp() } } class HubMemReq extends Bundle { @@ -80,14 +80,14 @@ class TransactionFinish extends Bundle { } class ioTileLink extends Bundle { - val xact_init = (new ioDecoupled) { new TransactionInit() }.flip - val xact_init_data = (new ioDecoupled) { new TransactionInitData() }.flip - val xact_abort = (new ioDecoupled) { new TransactionAbort() } - val probe_req = (new ioDecoupled) { new ProbeRequest() } - val probe_rep = (new ioDecoupled) { new ProbeReply() }.flip - val probe_rep_data = (new ioDecoupled) { new ProbeReplyData() }.flip - val xact_rep = (new ioValid) { new TransactionReply() } - val xact_finish = (new ioDecoupled) { new TransactionFinish() }.flip + val xact_init = (new ioDecoupled) { new TransactionInit() } + val xact_init_data = (new ioDecoupled) { new TransactionInitData() } + val xact_abort = (new ioDecoupled) { new TransactionAbort() }.flip + val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip + val probe_rep = (new ioDecoupled) { new ProbeReply() } + val probe_rep_data = (new ioDecoupled) { new ProbeReplyData() } + val xact_rep = (new ioPipe) { new TransactionReply() } + val xact_finish = (new ioDecoupled) { new TransactionFinish() } } object cpuCmdToRW { @@ -181,20 +181,20 @@ trait FourStateCoherence extends CoherencePolicy { class XactTracker(id: Int) extends Component with CoherencePolicy { val io = new Bundle { - val alloc_req = (new ioDecoupled) { new TrackerAllocReq() } + val alloc_req = (new ioDecoupled) { new TrackerAllocReq() }.flip val probe_data = (new TrackerProbeData).asInput val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) val p_rep_cnt_dec = Bits(NTILES, INPUT) val p_req_cnt_inc = Bits(NTILES, INPUT) - val p_rep_data = (new ioDecoupled) { new ProbeReplyData() } - val x_init_data = (new ioDecoupled) { new TransactionInitData() } + val p_rep_data = (new ioDecoupled) { new ProbeReplyData() }.flip + val x_init_data = (new ioDecoupled) { new TransactionInitData() }.flip val sent_x_rep_ack = Bool(INPUT) - val mem_req_cmd = (new ioDecoupled) { new MemReqCmd() }.flip - val mem_req_data = (new ioDecoupled) { new MemData() }.flip + val mem_req_cmd = (new ioDecoupled) { new MemReqCmd() } + val mem_req_data = (new ioDecoupled) { new MemData() } val mem_req_lock = Bool(OUTPUT) - val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip + val probe_req = (new ioDecoupled) { new ProbeRequest() } val busy = Bool(OUTPUT) val addr = Bits(PADDR_BITS, OUTPUT) val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) From 9ad4fd5814cd1578a7aa6de372e77ec5a9e08416 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 2 Mar 2012 12:19:21 -0800 Subject: [PATCH 034/374] BroadcastHub can be elaborated by C and vlsi backends --- uncore/coherence.scala | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 97d098bd..536a4775 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -198,9 +198,9 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { val busy = Bool(OUTPUT) val addr = Bits(PADDR_BITS, OUTPUT) val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) - val p_rep_tile_id = Bits(log2up(NTILES), INPUT) + val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) - val sharer_count = Bits(TILE_ID_BITS, OUTPUT) + val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) val t_type = Bits(TTYPE_BITS, OUTPUT) val push_p_req = Bits(NTILES, OUTPUT) val pop_p_rep = Bits(NTILES, OUTPUT) @@ -411,8 +411,8 @@ class CoherenceHubBroadcast extends CoherenceHub { val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_cnt_dec_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } - val p_req_cnt_inc_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } + val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(NTILES){ Wire(){Bool()} } } + val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(NTILES){ Wire(){Bool()} } } val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } for( i <- 0 until NGLOBAL_XACTS) { @@ -424,14 +424,16 @@ class CoherenceHubBroadcast extends CoherenceHub { t_type_arr(i) := t.t_type sh_count_arr(i) := t.sharer_count send_x_rep_ack_arr(i) := t.send_x_rep_ack - do_free_arr(i) := Bool(false) - p_rep_cnt_dec_arr(i) := Bits(0) - p_req_cnt_inc_arr(i) := Bits(0) - sent_x_rep_ack_arr(i) := Bool(false) t.xact_finish := do_free_arr(i) - t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i) - t.p_req_cnt_inc := p_req_cnt_inc_arr(i) + t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits + t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits t.sent_x_rep_ack := sent_x_rep_ack_arr(i) + do_free_arr(i) := Bool(false) + sent_x_rep_ack_arr(i) := Bool(false) + for( j <- 0 until NTILES) { + p_rep_cnt_dec_arr(i)(j) := Bool(false) + p_req_cnt_inc_arr(i)(j) := Bool(false) + } } // Free finished transactions @@ -495,8 +497,7 @@ class CoherenceHubBroadcast extends CoherenceHub { trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits for( j <- 0 until NTILES) { val p_rep = io.tiles(j).probe_rep - val dec = p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) - p_rep_cnt_dec_arr(UFix(i)) := p_rep_cnt_dec_arr(UFix(i)).bitSet(UFix(j), dec) + p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) } } @@ -556,7 +557,7 @@ class CoherenceHubBroadcast extends CoherenceHub { val t = trackerList(i).io p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits p_req_arb_arr(j).io.in(i).valid := t.probe_req.valid && t.push_p_req(j) - p_rep_cnt_dec_arr(i) = p_rep_cnt_dec_arr(i).bitSet(UFix(j), p_req_arb_arr(j).io.in(i).ready) + p_req_cnt_inc_arr(i)(j) := p_req_arb_arr(j).io.in(i).ready } p_req_arb_arr(j).io.out <> io.tiles(j).probe_req } From 6ad7aa5cdc586af0ec6a9744f0d6d7cc4cafa1f0 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 2 Mar 2012 16:18:32 -0800 Subject: [PATCH 035/374] flip direction of ioPipe to match ioDecoupled --- uncore/coherence.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 536a4775..195234a4 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -23,7 +23,7 @@ class ioMem() extends Bundle { val req_cmd = (new ioDecoupled) { new MemReqCmd() } val req_data = (new ioDecoupled) { new MemData() } - val resp = (new ioPipe) { new MemResp() } + val resp = (new ioPipe) { new MemResp() }.flip } class HubMemReq extends Bundle { @@ -86,7 +86,7 @@ class ioTileLink extends Bundle { val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip val probe_rep = (new ioDecoupled) { new ProbeReply() } val probe_rep_data = (new ioDecoupled) { new ProbeReplyData() } - val xact_rep = (new ioPipe) { new TransactionReply() } + val xact_rep = (new ioPipe) { new TransactionReply() }.flip val xact_finish = (new ioDecoupled) { new TransactionFinish() } } From 993341d27efcc486234716a1e1c45fedc3cf10fb Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 2 Mar 2012 13:41:01 -0800 Subject: [PATCH 036/374] Correction to probe reply w/ data handling --- uncore/coherence.scala | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 195234a4..638cc0d7 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -31,8 +31,7 @@ class HubMemReq extends Bundle { } class TrackerProbeData extends Bundle { - val valid = Bool() - val data_tile_id = Bits(width = log2up(NTILES)) + val tile_id = Bits(width = TILE_ID_BITS) } class TrackerAllocReq extends Bundle { @@ -182,7 +181,7 @@ trait FourStateCoherence extends CoherencePolicy { class XactTracker(id: Int) extends Component with CoherencePolicy { val io = new Bundle { val alloc_req = (new ioDecoupled) { new TrackerAllocReq() }.flip - val probe_data = (new TrackerProbeData).asInput + val p_data = (new ioPipe) { new TrackerProbeData() } val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) val p_rep_cnt_dec = Bits(NTILES, INPUT) @@ -331,9 +330,9 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { state := s_mem } } - when(io.probe_data.valid) { + when(io.p_data.valid) { p_rep_data_needs_write := Bool(true) - p_rep_tile_id_ := io.p_rep_tile_id + p_rep_tile_id_ := io.p_data.bits.tile_id } } is(s_mem) { @@ -413,7 +412,9 @@ class CoherenceHubBroadcast extends CoherenceHub { val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(NTILES){ Wire(){Bool()} } } val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(NTILES){ Wire(){Bool()} } } - val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } + val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bits(width = TILE_ID_BITS)} } + val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io @@ -425,14 +426,18 @@ class CoherenceHubBroadcast extends CoherenceHub { sh_count_arr(i) := t.sharer_count send_x_rep_ack_arr(i) := t.send_x_rep_ack t.xact_finish := do_free_arr(i) + t.p_data.bits.tile_id := p_data_tile_id_arr(i) + t.p_data.valid := p_data_valid_arr(i) t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits t.sent_x_rep_ack := sent_x_rep_ack_arr(i) do_free_arr(i) := Bool(false) sent_x_rep_ack_arr(i) := Bool(false) + p_data_tile_id_arr(i) := Bits(0, width = TILE_ID_BITS) + p_data_valid_arr(i) := Bool(false) for( j <- 0 until NTILES) { - p_rep_cnt_dec_arr(i)(j) := Bool(false) - p_req_cnt_inc_arr(i)(j) := Bool(false) + p_rep_cnt_dec_arr(i)(j) := Bool(false) + p_req_cnt_inc_arr(i)(j) := Bool(false) } } @@ -490,7 +495,9 @@ class CoherenceHubBroadcast extends CoherenceHub { val p_rep_data = io.tiles(j).probe_rep_data val idx = p_rep.bits.global_xact_id p_rep.ready := foldR(trackerList.map(_.io.pop_p_rep(j)))(_ || _) - p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) + p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) + p_data_valid_arr(idx) := p_rep.valid && p_rep.bits.has_data + p_data_tile_id_arr(idx) := UFix(j) } for( i <- 0 until NGLOBAL_XACTS ) { trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid From d76f6640070bd34819dc62ddbe8c131ac933ebd0 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 2 Mar 2012 21:58:50 -0800 Subject: [PATCH 037/374] Filled out 4 state coherence functions for cache --- uncore/coherence.scala | 44 ++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 638cc0d7..4532794b 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -156,16 +156,26 @@ trait FourStateCoherence extends CoherencePolicy { def newStateOnWriteback() = tileInvalid def newStateOnFlush() = tileInvalid + def newStateOnHit(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, tileExclusiveDirty, state) + } + def newStateOnTransactionRep(incoming: TransactionReply, outstanding: TransactionInit): UFix = { + MuxLookup(incoming.t_type, tileInvalid, Array( + X_READ_SHARED -> tileShared, + X_READ_EXCLUSIVE -> Mux(outstanding.t_type === X_READ_EXCLUSIVE, tileExclusiveDirty, tileExclusiveClean), + X_READ_EXCLUSIVE_ACK -> tileExclusiveDirty, + X_READ_UNCACHED -> tileInvalid, + X_WRITE_UNCACHED -> tileInvalid + )) + } + def needsSecondaryXact(cmd: Bits, outstanding: TransactionInit): Bool = { + val (read, write) = cpuCmdToRW(cmd) + (read && (outstanding.t_type === X_READ_UNCACHED || outstanding.t_type === X_WRITE_UNCACHED)) || + (write && (outstanding.t_type != X_READ_EXCLUSIVE)) + } - // TODO: New funcs as compared to incoherent protocol: - def newState(cmd: Bits, state: UFix): UFix - def newStateOnHit(cmd: Bits, state: UFix): UFix - def newStateOnPrimaryMiss(cmd: Bits): UFix - def newStateOnSecondaryMiss(cmd: Bits, state: UFix): UFix - - def needsSecondaryXact (cmd: Bits, outstanding: TransactionInit): Bool - - def newStateOnProbe (incoming: ProbeRequest, state: UFix): Bits = { + def newStateOnProbeReq(incoming: ProbeRequest, state: UFix): Bits = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeDowngrade -> tileShared, @@ -174,26 +184,26 @@ trait FourStateCoherence extends CoherencePolicy { } def replyTypeHasData (reply: TransactionReply): Bool = { - (reply.t_type != X_WRITE_UNCACHED) + (reply.t_type != X_WRITE_UNCACHED && reply.t_type != X_READ_EXCLUSIVE_ACK) } } class XactTracker(id: Int) extends Component with CoherencePolicy { val io = new Bundle { - val alloc_req = (new ioDecoupled) { new TrackerAllocReq() }.flip - val p_data = (new ioPipe) { new TrackerProbeData() } + val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip + val p_data = (new ioPipe) { new TrackerProbeData } val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) val p_rep_cnt_dec = Bits(NTILES, INPUT) val p_req_cnt_inc = Bits(NTILES, INPUT) - val p_rep_data = (new ioDecoupled) { new ProbeReplyData() }.flip - val x_init_data = (new ioDecoupled) { new TransactionInitData() }.flip + val p_rep_data = (new ioDecoupled) { new ProbeReplyData }.flip + val x_init_data = (new ioDecoupled) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) - val mem_req_cmd = (new ioDecoupled) { new MemReqCmd() } - val mem_req_data = (new ioDecoupled) { new MemData() } + val mem_req_cmd = (new ioDecoupled) { new MemReqCmd } + val mem_req_data = (new ioDecoupled) { new MemData } val mem_req_lock = Bool(OUTPUT) - val probe_req = (new ioDecoupled) { new ProbeRequest() } + val probe_req = (new ioDecoupled) { new ProbeRequest } val busy = Bool(OUTPUT) val addr = Bits(PADDR_BITS, OUTPUT) val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) From 7846d5e01d8a1cf86895e56589b98946572cccf9 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 2 Mar 2012 23:51:53 -0800 Subject: [PATCH 038/374] Removed has_data fields from all coherence messages, increased message type names to compensate --- uncore/coherence.scala | 78 +++++++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 35 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 4532794b..0ce1f1c9 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -42,8 +42,7 @@ class TrackerAllocReq extends Bundle { class TransactionInit extends Bundle { - val t_type = Bits(width = TTYPE_BITS) - val has_data = Bool() + val t_type = Bits(width = X_INIT_TYPE_BITS) val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) val address = UFix(width = PADDR_BITS) } @@ -55,21 +54,20 @@ class TransactionAbort extends Bundle { } class ProbeRequest extends Bundle { - val p_type = Bits(width = PTYPE_BITS) + val p_type = Bits(width = P_REQ_TYPE_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) val address = Bits(width = PADDR_BITS) } class ProbeReply extends Bundle { - val p_type = Bits(width = PTYPE_BITS) - val has_data = Bool() + val p_type = Bits(width = P_REP_TYPE_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } class ProbeReplyData extends MemData class TransactionReply extends MemData { - val t_type = Bits(width = TTYPE_BITS) + val t_type = Bits(width = X_REP_TYPE_BITS) val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } @@ -162,17 +160,17 @@ trait FourStateCoherence extends CoherencePolicy { } def newStateOnTransactionRep(incoming: TransactionReply, outstanding: TransactionInit): UFix = { MuxLookup(incoming.t_type, tileInvalid, Array( - X_READ_SHARED -> tileShared, - X_READ_EXCLUSIVE -> Mux(outstanding.t_type === X_READ_EXCLUSIVE, tileExclusiveDirty, tileExclusiveClean), - X_READ_EXCLUSIVE_ACK -> tileExclusiveDirty, - X_READ_UNCACHED -> tileInvalid, - X_WRITE_UNCACHED -> tileInvalid + X_REP_READ_SHARED -> tileShared, + X_REP_READ_EXCLUSIVE -> Mux(outstanding.t_type === X_INIT_READ_EXCLUSIVE, tileExclusiveDirty, tileExclusiveClean), + X_REP_READ_EXCLUSIVE_ACK -> tileExclusiveDirty, + X_REP_READ_UNCACHED -> tileInvalid, + X_REP_WRITE_UNCACHED -> tileInvalid )) } def needsSecondaryXact(cmd: Bits, outstanding: TransactionInit): Bool = { val (read, write) = cpuCmdToRW(cmd) - (read && (outstanding.t_type === X_READ_UNCACHED || outstanding.t_type === X_WRITE_UNCACHED)) || - (write && (outstanding.t_type != X_READ_EXCLUSIVE)) + (read && (outstanding.t_type === X_INIT_READ_UNCACHED || outstanding.t_type === X_INIT_WRITE_UNCACHED)) || + (write && (outstanding.t_type != X_INIT_READ_EXCLUSIVE)) } def newStateOnProbeReq(incoming: ProbeRequest, state: UFix): Bits = { @@ -183,12 +181,20 @@ trait FourStateCoherence extends CoherencePolicy { )) } - def replyTypeHasData (reply: TransactionReply): Bool = { - (reply.t_type != X_WRITE_UNCACHED && reply.t_type != X_READ_EXCLUSIVE_ACK) + def probeReplyHasData (reply: ProbeReply): Bool = { + (reply.p_type === P_REP_INVALIDATE_DATA || + reply.p_type === P_REP_DOWNGRADE_DATA || + reply.p_type === P_REP_COPY_DATA) + } + def transactionInitHasData (init: TransactionInit): Bool = { + (init.t_type != X_INIT_WRITE_UNCACHED) + } + def transactionReplyHasData (reply: TransactionReply): Bool = { + (reply.t_type != X_REP_WRITE_UNCACHED && reply.t_type != X_REP_READ_EXCLUSIVE_ACK) } } -class XactTracker(id: Int) extends Component with CoherencePolicy { +class XactTracker(id: Int) extends Component with FourStateCoherence { val io = new Bundle { val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip val p_data = (new ioPipe) { new TrackerProbeData } @@ -210,7 +216,7 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) - val t_type = Bits(TTYPE_BITS, OUTPUT) + val t_type = Bits(X_INIT_TYPE_BITS, OUTPUT) val push_p_req = Bits(NTILES, OUTPUT) val pop_p_rep = Bits(NTILES, OUTPUT) val pop_p_rep_data = Bits(NTILES, OUTPUT) @@ -220,18 +226,20 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { } def sendProbeReqType(t_type: UFix, global_state: UFix): UFix = { - MuxCase(P_COPY, Array((t_type === X_READ_SHARED) -> P_DOWNGRADE, - (t_type === X_READ_EXCLUSIVE) -> P_INVALIDATE, - (t_type === X_READ_UNCACHED) -> P_COPY, - (t_type === X_WRITE_UNCACHED) -> P_INVALIDATE)) + MuxLookup(t_type, P_REQ_COPY, Array( + X_INIT_READ_SHARED -> P_REQ_DOWNGRADE, + X_INIT_READ_EXCLUSIVE -> P_REQ_INVALIDATE, + X_INIT_READ_UNCACHED -> P_REQ_COPY, + X_INIT_WRITE_UNCACHED -> P_REQ_INVALIDATE + )) } def needsMemRead(t_type: UFix, global_state: UFix): Bool = { - (t_type != X_WRITE_UNCACHED) + (t_type != X_INIT_WRITE_UNCACHED) } def needsAckRep(t_type: UFix, global_state: UFix): Bool = { - (t_type === X_WRITE_UNCACHED) + (t_type === X_INIT_WRITE_UNCACHED) } val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } @@ -312,7 +320,7 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { t_type_ := io.alloc_req.bits.xact_init.t_type init_tile_id_ := io.alloc_req.bits.init_tile_id tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id - x_init_data_needs_write := io.alloc_req.bits.xact_init.has_data + x_init_data_needs_write := transactionInitHasData(io.alloc_req.bits.xact_init) x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) p_rep_count := UFix(NTILES-1) p_req_flags := ~( UFix(1) << io.alloc_req.bits.init_tile_id ) @@ -378,7 +386,7 @@ abstract class CoherenceHub extends Component with CoherencePolicy { class CoherenceHubNull extends CoherenceHub { val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.t_type === X_WRITE_UNCACHED + val is_write = x_init.bits.t_type === X_INIT_WRITE_UNCACHED x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write @@ -387,7 +395,7 @@ class CoherenceHubNull extends CoherenceHub { io.mem.req_data <> io.tiles(0).xact_init_data val x_rep = io.tiles(0).xact_rep - x_rep.bits.t_type := Mux(io.mem.resp.valid, X_READ_EXCLUSIVE, X_WRITE_UNCACHED) + x_rep.bits.t_type := Mux(io.mem.resp.valid, X_REP_READ_EXCLUSIVE, X_REP_WRITE_UNCACHED) x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.bits.data @@ -395,17 +403,17 @@ class CoherenceHubNull extends CoherenceHub { } -class CoherenceHubBroadcast extends CoherenceHub { +class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ def coherenceConflict(addr1: Bits, addr2: Bits): Bool = { addr1(PADDR_BITS-1, OFFSET_BITS) === addr2(PADDR_BITS-1, OFFSET_BITS) } def getTransactionReplyType(t_type: UFix, count: UFix): Bits = { - MuxLookup(t_type, X_READ_UNCACHED, Array( - X_READ_SHARED -> Mux(count > UFix(0), X_READ_SHARED, X_READ_EXCLUSIVE), - X_READ_EXCLUSIVE -> X_READ_EXCLUSIVE, - X_READ_UNCACHED -> X_READ_UNCACHED, - X_WRITE_UNCACHED -> X_WRITE_UNCACHED + MuxLookup(t_type, X_REP_READ_UNCACHED, Array( + X_INIT_READ_SHARED -> Mux(count > UFix(0), X_REP_READ_SHARED, X_REP_READ_EXCLUSIVE), + X_INIT_READ_EXCLUSIVE -> X_REP_READ_EXCLUSIVE, + X_INIT_READ_UNCACHED -> X_REP_READ_UNCACHED, + X_INIT_WRITE_UNCACHED -> X_REP_WRITE_UNCACHED )) } @@ -415,7 +423,7 @@ class CoherenceHubBroadcast extends CoherenceHub { val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TTYPE_BITS)} } + val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } @@ -506,7 +514,7 @@ class CoherenceHubBroadcast extends CoherenceHub { val idx = p_rep.bits.global_xact_id p_rep.ready := foldR(trackerList.map(_.io.pop_p_rep(j)))(_ || _) p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) - p_data_valid_arr(idx) := p_rep.valid && p_rep.bits.has_data + p_data_valid_arr(idx) := p_rep.valid && probeReplyHasData(p_rep.bits) p_data_tile_id_arr(idx) := UFix(j) } for( i <- 0 until NGLOBAL_XACTS ) { @@ -527,7 +535,7 @@ class CoherenceHubBroadcast extends CoherenceHub { for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io conflicts(UFix(i), t.busy && coherenceConflict(t.addr, x_init.bits.address) && - !(x_init.bits.has_data && (UFix(j) === t.init_tile_id))) + !(transactionInitHasData(x_init.bits) && (UFix(j) === t.init_tile_id))) // Don't abort writebacks stalled on mem. // TODO: This assumes overlapped writeback init reqs to // the same addr will never be issued; is this ok? From 802f857cb69bb00086e841ddaddfca135669a8bb Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Sat, 3 Mar 2012 15:07:22 -0800 Subject: [PATCH 039/374] fix store prefetch bug, it no longer occupies an entry in the sdq --- uncore/coherence.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 0ce1f1c9..005f17c9 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -92,8 +92,8 @@ object cpuCmdToRW { val store = (cmd === M_XWR) val load = (cmd === M_XRD) val amo = cmd(3).toBool - val read = load || amo || (cmd === M_PFR) - val write = store || amo || (cmd === M_PFW) + val read = load || amo || (cmd === M_PFR) || (cmd === M_PFW) + val write = store || amo (read, write) } } From a49625c114b5f108451a7c8e64c68cbb36569c34 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 5 Mar 2012 16:34:27 -0800 Subject: [PATCH 040/374] Broadcast hub control logic bugfixes and code cleanup --- uncore/coherence.scala | 81 +++++++++++++++++++----------------------- 1 file changed, 37 insertions(+), 44 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 005f17c9..c6e702cd 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -26,18 +26,13 @@ class ioMem() extends Bundle val resp = (new ioPipe) { new MemResp() }.flip } -class HubMemReq extends Bundle { - val lock = Bool() -} - class TrackerProbeData extends Bundle { val tile_id = Bits(width = TILE_ID_BITS) } class TrackerAllocReq extends Bundle { val xact_init = new TransactionInit() - val init_tile_id = Bits(width = TILE_ID_BITS) - val data_valid = Bool() + val tile_id = Bits(width = TILE_ID_BITS) } @@ -98,8 +93,7 @@ object cpuCmdToRW { } } -trait CoherencePolicy { -} +trait CoherencePolicy { } trait ThreeStateIncoherence extends CoherencePolicy { val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } @@ -242,36 +236,18 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { (t_type === X_INIT_WRITE_UNCACHED) } - val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } - val state = Reg(resetVal = s_idle) - val addr_ = Reg{ UFix() } - val t_type_ = Reg{ Bits() } - val init_tile_id_ = Reg{ Bits() } - val tile_xact_id_ = Reg{ Bits() } - val p_rep_count = Reg(resetVal = UFix(0, width = log2up(NTILES))) - val p_req_flags = Reg(resetVal = Bits(0, width = NTILES)) - val p_rep_tile_id_ = Reg{ Bits() } - val x_needs_read = Reg(resetVal = Bool(false)) - val x_init_data_needs_write = Reg(resetVal = Bool(false)) - val p_rep_data_needs_write = Reg(resetVal = Bool(false)) - val mem_cmd_sent = Reg(resetVal = Bool(false)) - val mem_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) - val mem_cnt_next = mem_cnt + UFix(1) - - def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioDecoupled[MemData], trigger: Bool, pop: Bool) { - req_cmd.valid := mem_cmd_sent + def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioDecoupled[MemData], trigger: Bool, pop_data: Bool, cmd_sent: Bool) { + req_cmd.valid := !cmd_sent req_cmd.bits.rw := Bool(true) - //TODO: why does req_data <> data segfault? - req_data.valid := data.valid - req_data.bits.data := data.bits.data data.ready := req_data.ready + req_data <> data lock := Bool(true) when(req_cmd.ready && req_cmd.valid) { - mem_cmd_sent := Bool(false) + cmd_sent := Bool(true) } when(req_data.ready && req_data.valid) { - pop := Bool(true) - mem_cnt := mem_cnt_next + pop_data := Bool(true) + mem_cnt := mem_cnt_next } when(mem_cnt === ~UFix(0)) { trigger := Bool(false) @@ -286,6 +262,25 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { } } + val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } + val state = Reg(resetVal = s_idle) + val addr_ = Reg{ UFix() } + val t_type_ = Reg{ Bits() } + val init_tile_id_ = Reg{ Bits() } + val tile_xact_id_ = Reg{ Bits() } + val p_rep_count = Reg(resetVal = UFix(0, width = log2up(NTILES))) + val p_req_flags = Reg(resetVal = Bits(0, width = NTILES)) + val p_rep_tile_id_ = Reg{ Bits() } + val x_needs_read = Reg(resetVal = Bool(false)) + val x_init_data_needs_write = Reg(resetVal = Bool(false)) + val p_rep_data_needs_write = Reg(resetVal = Bool(false)) + val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val mem_cmd_sent = Reg(resetVal = Bool(false)) + val mem_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val mem_cnt_next = mem_cnt + UFix(1) + val mem_cnt_max = ~UFix(0, width = log2up(REFILL_CYCLES)) + io.busy := state != s_idle io.addr := addr_ io.init_tile_id := init_tile_id_ @@ -318,16 +313,16 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { when( io.alloc_req.valid && io.can_alloc ) { addr_ := io.alloc_req.bits.xact_init.address t_type_ := io.alloc_req.bits.xact_init.t_type - init_tile_id_ := io.alloc_req.bits.init_tile_id + init_tile_id_ := io.alloc_req.bits.tile_id tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := transactionInitHasData(io.alloc_req.bits.xact_init) x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) p_rep_count := UFix(NTILES-1) - p_req_flags := ~( UFix(1) << io.alloc_req.bits.init_tile_id ) - state := Mux(p_req_flags.orR, s_probe, s_mem) + p_req_flags := ~( UFix(1) << io.alloc_req.bits.tile_id ) mem_cnt := UFix(0) mem_cmd_sent := Bool(false) io.pop_x_init := Bool(true) + state := Mux(p_req_flags.orR, s_probe, s_mem) } } is(s_probe) { @@ -342,9 +337,8 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { val p_rep_count_next = p_rep_count - PopCount(io.p_rep_cnt_dec) io.pop_p_rep := io.p_rep_cnt_dec p_rep_count := p_rep_count_next - when(p_rep_count_next === UFix(0)) { - mem_cnt := UFix(0) - mem_cmd_sent := Bool(false) + when(p_rep_count === UFix(0)) { + io.pop_p_rep := Bool(true) state := s_mem } } @@ -355,9 +349,9 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { } is(s_mem) { when (p_rep_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, io.p_rep_data, p_rep_data_needs_write, io.pop_p_rep_data) + doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, io.p_rep_data, p_rep_data_needs_write, io.pop_p_rep_data, p_w_mem_cmd_sent) } . elsewhen(x_init_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, io.x_init_data, x_init_data_needs_write, io.pop_x_init_data) + doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, io.x_init_data, x_init_data_needs_write, io.pop_x_init_data, x_w_mem_cmd_sent) } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { @@ -565,10 +559,9 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ val x_init_data = io.tiles(j).xact_init_data init_arb.io.in(j).valid := x_init.valid init_arb.io.in(j).bits.xact_init := x_init.bits - init_arb.io.in(j).bits.init_tile_id := UFix(j) - init_arb.io.in(j).bits.data_valid := x_init_data.valid - x_init.ready := aborting(j) || foldR(trackerList.map(_.io.pop_x_init && init_arb.io.out.bits.init_tile_id === UFix(j)))(_||_) - x_init_data.ready := aborting(j) || foldR(trackerList.map(_.io.pop_x_init_data && init_arb.io.out.bits.init_tile_id === UFix(j)))(_||_) + init_arb.io.in(j).bits.tile_id := UFix(j) + x_init.ready := aborting(j) || foldR(trackerList.map(_.io.pop_x_init && init_arb.io.out.bits.tile_id === UFix(j)))(_||_) + x_init_data.ready := aborting(j) || foldR(trackerList.map(_.io.pop_x_init_data && init_arb.io.out.bits.tile_id === UFix(j)))(_||_) } alloc_arb.io.out.ready := init_arb.io.out.valid && !busy_arr.toBits.andR && From 82822dfeecd8a5ed3fe9090c3fad56d52ea92dc4 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 5 Mar 2012 17:44:30 -0800 Subject: [PATCH 041/374] Added aborted data dequeueing state machine for BroadcastHub --- uncore/coherence.scala | 58 +++++++++++++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 15 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index c6e702cd..fc4f3570 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -240,7 +240,8 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { req_cmd.valid := !cmd_sent req_cmd.bits.rw := Bool(true) data.ready := req_data.ready - req_data <> data + req_data.bits := data.bits + req_data.valid := data.valid lock := Bool(true) when(req_cmd.ready && req_cmd.valid) { cmd_sent := Bool(true) @@ -521,23 +522,51 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ } // Nack conflicting transaction init attempts - val aborting = Bits(0, width = NTILES) + val s_idle :: s_abort_drain :: s_abort_send :: s_abort_complete :: Nil = Enum(4){ UFix() } + val abort_state_arr = Vec(NTILES) { Reg(resetVal = s_idle) } + val want_to_abort_arr = Vec(NTILES) { Wire() { Bool()} } for( j <- 0 until NTILES ) { val x_init = io.tiles(j).xact_init + val x_init_data = io.tiles(j).xact_init_data val x_abort = io.tiles(j).xact_abort val conflicts = Bits(width = NGLOBAL_XACTS) for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(UFix(i), t.busy && coherenceConflict(t.addr, x_init.bits.address) && - !(transactionInitHasData(x_init.bits) && (UFix(j) === t.init_tile_id))) - // Don't abort writebacks stalled on mem. - // TODO: This assumes overlapped writeback init reqs to - // the same addr will never be issued; is this ok? + conflicts(UFix(i), t.busy && x_init.valid && coherenceConflict(t.addr, x_init.bits.address)) } x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id - val want_to_abort = conflicts.orR || busy_arr.toBits.andR - x_abort.valid := want_to_abort && x_init.valid - aborting.bitSet(UFix(j), want_to_abort && x_abort.ready) + val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + want_to_abort_arr(j) := conflicts.orR || busy_arr.toBits.andR + + x_abort.valid := Bool(false) + switch(abort_state_arr(j)) { + is(s_idle) { + when(want_to_abort_arr(j)) { + when(transactionInitHasData(x_init.bits)) { + abort_state_arr(j) := s_abort_drain + } . otherwise { + abort_state_arr(j) := s_abort_send + } + } + } + is(s_abort_drain) { // raises x_init_data.ready below + when(x_init_data.valid) { + abort_cnt := abort_cnt + UFix(1) + } + when(abort_cnt === ~UFix(0, width = log2up(REFILL_CYCLES))) { + abort_state_arr(j) := s_abort_send + } + } + is(s_abort_send) { // nothing is dequeued for now + x_abort.valid := Bool(true) + when(x_abort.ready) { + abort_state_arr(j) := s_abort_complete + } + } + is(s_abort_complete) { // raises x_init.ready below + abort_state_arr(j) := s_idle + } + } } // Handle transaction initiation requests @@ -557,15 +586,14 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ for( j <- 0 until NTILES ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data - init_arb.io.in(j).valid := x_init.valid + init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid init_arb.io.in(j).bits.xact_init := x_init.bits init_arb.io.in(j).bits.tile_id := UFix(j) - x_init.ready := aborting(j) || foldR(trackerList.map(_.io.pop_x_init && init_arb.io.out.bits.tile_id === UFix(j)))(_||_) - x_init_data.ready := aborting(j) || foldR(trackerList.map(_.io.pop_x_init_data && init_arb.io.out.bits.tile_id === UFix(j)))(_||_) + x_init.ready := (abort_state_arr(j) === s_abort_complete) || foldR(trackerList.map(_.io.pop_x_init && init_arb.io.out.bits.tile_id === UFix(j)))(_||_) + x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data && init_arb.io.out.bits.tile_id === UFix(j)))(_||_) } - alloc_arb.io.out.ready := init_arb.io.out.valid && !busy_arr.toBits.andR && - !foldR(trackerList.map(t => t.io.busy && coherenceConflict(t.io.addr, init_arb.io.out.bits.xact_init.address)))(_||_) + alloc_arb.io.out.ready := init_arb.io.out.valid // Handle probe request generation // Must arbitrate for each request port From 1024a460f238ff6df738c0d7e119f1ba9d0234be Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 6 Mar 2012 00:31:44 -0800 Subject: [PATCH 042/374] support memory transaction aborts --- uncore/coherence.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index fc4f3570..7bcd19c3 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -395,6 +395,8 @@ class CoherenceHubNull extends CoherenceHub { x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.bits.data x_rep.valid := io.mem.resp.valid || x_init.valid && is_write + + io.tiles(0).xact_abort.valid := Bool(false) } From 012b991ffead7aa724adf6565f6ced61e7c9f034 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 6 Mar 2012 15:47:19 -0800 Subject: [PATCH 043/374] implement transaction finish messages --- uncore/coherence.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 7bcd19c3..817e6d73 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -397,6 +397,7 @@ class CoherenceHubNull extends CoherenceHub { x_rep.valid := io.mem.resp.valid || x_init.valid && is_write io.tiles(0).xact_abort.valid := Bool(false) + io.tiles(0).xact_finish.ready := Bool(true) } From 3874da4a4ffa304688a0b708322160f48e8b8fd6 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 6 Mar 2012 13:49:20 -0800 Subject: [PATCH 044/374] Added store dependency queues to BroadcastHub. Minor improvements to utils. --- uncore/coherence.scala | 80 ++++++++++++++++++++++++++++++++---------- 1 file changed, 62 insertions(+), 18 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 817e6d73..51505882 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -35,6 +35,9 @@ class TrackerAllocReq extends Bundle { val tile_id = Bits(width = TILE_ID_BITS) } +class TrackerDependency extends Bundle { + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) +} class TransactionInit extends Bundle { val t_type = Bits(width = X_INIT_TYPE_BITS) @@ -196,9 +199,11 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { val xact_finish = Bool(INPUT) val p_rep_cnt_dec = Bits(NTILES, INPUT) val p_req_cnt_inc = Bits(NTILES, INPUT) - val p_rep_data = (new ioDecoupled) { new ProbeReplyData }.flip - val x_init_data = (new ioDecoupled) { new TransactionInitData }.flip + val p_rep_data = (new ioPipe) { new ProbeReplyData }.flip + val x_init_data = (new ioPipe) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) + val p_rep_data_dep = (new ioPipe) { new TrackerDependency }.flip + val x_init_data_dep = (new ioPipe) { new TrackerDependency }.flip val mem_req_cmd = (new ioDecoupled) { new MemReqCmd } val mem_req_data = (new ioDecoupled) { new MemData } @@ -214,8 +219,10 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { val push_p_req = Bits(NTILES, OUTPUT) val pop_p_rep = Bits(NTILES, OUTPUT) val pop_p_rep_data = Bits(NTILES, OUTPUT) + val pop_p_rep_dep = Bits(NTILES, OUTPUT) val pop_x_init = Bool(OUTPUT) val pop_x_init_data = Bool(OUTPUT) + val pop_x_init_dep = Bits(NTILES, OUTPUT) val send_x_rep_ack = Bool(OUTPUT) } @@ -236,13 +243,12 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { (t_type === X_INIT_WRITE_UNCACHED) } - def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioDecoupled[MemData], trigger: Bool, pop_data: Bool, cmd_sent: Bool) { - req_cmd.valid := !cmd_sent + def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioPipe[MemData], trigger: Bool, pop_data: Bool, cmd_sent: Bool, pop_dep: Bool, at_front_of_dep_queue: Bool) { + req_cmd.valid := !cmd_sent && at_front_of_dep_queue req_cmd.bits.rw := Bool(true) - data.ready := req_data.ready + req_data.valid := data.valid && at_front_of_dep_queue req_data.bits := data.bits - req_data.valid := data.valid - lock := Bool(true) + lock := at_front_of_dep_queue when(req_cmd.ready && req_cmd.valid) { cmd_sent := Bool(true) } @@ -250,7 +256,8 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { pop_data := Bool(true) mem_cnt := mem_cnt_next } - when(mem_cnt === ~UFix(0)) { + when(mem_cnt_next === UFix(0)) { + pop_dep := Bool(true) trigger := Bool(false) } } @@ -277,7 +284,6 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { val p_rep_data_needs_write = Reg(resetVal = Bool(false)) val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val mem_cmd_sent = Reg(resetVal = Bool(false)) val mem_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2up(REFILL_CYCLES)) @@ -285,6 +291,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { io.busy := state != s_idle io.addr := addr_ io.init_tile_id := init_tile_id_ + io.p_rep_tile_id := p_rep_tile_id_ io.tile_xact_id := tile_xact_id_ io.sharer_count := UFix(NTILES) // TODO: Broadcast only io.t_type := t_type_ @@ -303,11 +310,11 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { io.push_p_req := Bits(0, width = NTILES) io.pop_p_rep := Bits(0, width = NTILES) io.pop_p_rep_data := Bits(0, width = NTILES) + io.pop_p_rep_dep := Bits(0, width = NTILES) io.pop_x_init := Bool(false) io.pop_x_init_data := Bool(false) + io.pop_x_init_dep := Bits(0, width = NTILES) io.send_x_rep_ack := Bool(false) - io.x_init_data.ready := Bool(false) // don't care - io.p_rep_data.ready := Bool(false) // don't care switch (state) { is(s_idle) { @@ -321,7 +328,8 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { p_rep_count := UFix(NTILES-1) p_req_flags := ~( UFix(1) << io.alloc_req.bits.tile_id ) mem_cnt := UFix(0) - mem_cmd_sent := Bool(false) + p_w_mem_cmd_sent := Bool(false) + x_w_mem_cmd_sent := Bool(false) io.pop_x_init := Bool(true) state := Mux(p_req_flags.orR, s_probe, s_mem) } @@ -350,9 +358,25 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { } is(s_mem) { when (p_rep_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, io.p_rep_data, p_rep_data_needs_write, io.pop_p_rep_data, p_w_mem_cmd_sent) + doMemReqWrite(io.mem_req_cmd, + io.mem_req_data, + io.mem_req_lock, + io.p_rep_data, + p_rep_data_needs_write, + io.pop_p_rep_data, + p_w_mem_cmd_sent, + io.pop_p_rep_dep, + io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id))) } . elsewhen(x_init_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, io.x_init_data, x_init_data_needs_write, io.pop_x_init_data, x_w_mem_cmd_sent) + doMemReqWrite(io.mem_req_cmd, + io.mem_req_data, + io.mem_req_lock, + io.x_init_data, + x_init_data_needs_write, + io.pop_x_init_data, + x_w_mem_cmd_sent, + io.pop_x_init_dep, + io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id))) } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { @@ -457,6 +481,9 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ } } + val p_rep_data_dep_list = List.fill(NTILES)((new queue(NGLOBAL_XACTS, true)){new TrackerDependency}) // depth must >= NPRIMARY + val x_init_data_dep_list = List.fill(NTILES)((new queue(NGLOBAL_XACTS, true)){new TrackerDependency}) // depth should >= NPRIMARY + // Free finished transactions for( j <- 0 until NTILES ) { val finish = io.tiles(j).xact_finish @@ -510,14 +537,23 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ val p_rep = io.tiles(j).probe_rep val p_rep_data = io.tiles(j).probe_rep_data val idx = p_rep.bits.global_xact_id - p_rep.ready := foldR(trackerList.map(_.io.pop_p_rep(j)))(_ || _) + val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) + val do_pop = foldR(pop_p_reps)(_ || _) + p_rep.ready := do_pop + p_rep_data_dep_list(j).io.enq.valid := do_pop + p_rep_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_p_reps) p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) p_data_valid_arr(idx) := p_rep.valid && probeReplyHasData(p_rep.bits) p_data_tile_id_arr(idx) := UFix(j) + p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) } for( i <- 0 until NGLOBAL_XACTS ) { trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits + + trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until NTILES).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) + trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until NTILES).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) + for( j <- 0 until NTILES) { val p_rep = io.tiles(j).probe_rep p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) @@ -532,14 +568,14 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_abort = io.tiles(j).xact_abort + val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) val conflicts = Bits(width = NGLOBAL_XACTS) for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io conflicts(UFix(i), t.busy && x_init.valid && coherenceConflict(t.addr, x_init.bits.address)) } x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id - val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) - want_to_abort_arr(j) := conflicts.orR || busy_arr.toBits.andR + want_to_abort_arr(j) := conflicts.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && transactionInitHasData(x_init.bits)) x_abort.valid := Bool(false) switch(abort_state_arr(j)) { @@ -585,6 +621,9 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid + //TODO trackerList(i).io.x_init_data_dep <> x_init_data_dep_arr(trackerList(i).io.init_tile_id) + trackerList(i).io.x_init_data_dep.bits <> MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until NTILES).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) + trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until NTILES).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) } for( j <- 0 until NTILES ) { val x_init = io.tiles(j).xact_init @@ -592,8 +631,13 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid init_arb.io.in(j).bits.xact_init := x_init.bits init_arb.io.in(j).bits.tile_id := UFix(j) - x_init.ready := (abort_state_arr(j) === s_abort_complete) || foldR(trackerList.map(_.io.pop_x_init && init_arb.io.out.bits.tile_id === UFix(j)))(_||_) + val pop_x_inits = trackerList.map(_.io.pop_x_init && init_arb.io.out.bits.tile_id === UFix(j)) + val do_pop = foldR(pop_x_inits)(_||_) + x_init_data_dep_list(j).io.enq.valid := do_pop + x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) + x_init.ready := (abort_state_arr(j) === s_abort_complete) || do_pop x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data && init_arb.io.out.bits.tile_id === UFix(j)))(_||_) + x_init_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) } alloc_arb.io.out.ready := init_arb.io.out.valid From 762f2551a70ba40ea8f527522d48551ecc8994a8 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 6 Mar 2012 15:54:19 -0800 Subject: [PATCH 045/374] newTransactionOnMiss() --- uncore/coherence.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 51505882..5a52ac44 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -155,6 +155,10 @@ trait FourStateCoherence extends CoherencePolicy { val (read, write) = cpuCmdToRW(cmd) Mux(write, tileExclusiveDirty, state) } + def newTransactionOnMiss(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) + } def newStateOnTransactionRep(incoming: TransactionReply, outstanding: TransactionInit): UFix = { MuxLookup(incoming.t_type, tileInvalid, Array( X_REP_READ_SHARED -> tileShared, From 1e02618cc800bea9c2de5231db628f38ab5ddb62 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 6 Mar 2012 17:01:47 -0800 Subject: [PATCH 046/374] hub code cleanup --- uncore/coherence.scala | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 5a52ac44..a04b7096 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -141,6 +141,9 @@ trait FourStateCoherence extends CoherencePolicy { (write && (state === tileExclusiveClean || state === tileExclusiveDirty))) } + //TODO: do we need isPresent() for determining that a line needs to be + //upgraded but that no replacement is needed? + def isValid (state: UFix): Bool = { state != tileInvalid } @@ -224,8 +227,8 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { val pop_p_rep = Bits(NTILES, OUTPUT) val pop_p_rep_data = Bits(NTILES, OUTPUT) val pop_p_rep_dep = Bits(NTILES, OUTPUT) - val pop_x_init = Bool(OUTPUT) - val pop_x_init_data = Bool(OUTPUT) + val pop_x_init = Bits(NTILES, OUTPUT) + val pop_x_init_data = Bits(NTILES, OUTPUT) val pop_x_init_dep = Bits(NTILES, OUTPUT) val send_x_rep_ack = Bool(OUTPUT) } @@ -247,7 +250,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { (t_type === X_INIT_WRITE_UNCACHED) } - def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioPipe[MemData], trigger: Bool, pop_data: Bool, cmd_sent: Bool, pop_dep: Bool, at_front_of_dep_queue: Bool) { + def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioPipe[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { req_cmd.valid := !cmd_sent && at_front_of_dep_queue req_cmd.bits.rw := Bool(true) req_data.valid := data.valid && at_front_of_dep_queue @@ -257,11 +260,11 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { cmd_sent := Bool(true) } when(req_data.ready && req_data.valid) { - pop_data := Bool(true) + pop_data := UFix(1) << tile_id mem_cnt := mem_cnt_next } when(mem_cnt_next === UFix(0)) { - pop_dep := Bool(true) + pop_dep := UFix(1) << tile_id trigger := Bool(false) } } @@ -315,8 +318,8 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { io.pop_p_rep := Bits(0, width = NTILES) io.pop_p_rep_data := Bits(0, width = NTILES) io.pop_p_rep_dep := Bits(0, width = NTILES) - io.pop_x_init := Bool(false) - io.pop_x_init_data := Bool(false) + io.pop_x_init := Bits(0, width = NTILES) + io.pop_x_init_data := Bits(0, width = NTILES) io.pop_x_init_dep := Bits(0, width = NTILES) io.send_x_rep_ack := Bool(false) @@ -334,7 +337,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) - io.pop_x_init := Bool(true) + io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id state := Mux(p_req_flags.orR, s_probe, s_mem) } } @@ -367,20 +370,22 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { io.mem_req_lock, io.p_rep_data, p_rep_data_needs_write, - io.pop_p_rep_data, p_w_mem_cmd_sent, + io.pop_p_rep_data, io.pop_p_rep_dep, - io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id))) + io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id)), + p_rep_tile_id_) } . elsewhen(x_init_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, io.x_init_data, x_init_data_needs_write, - io.pop_x_init_data, x_w_mem_cmd_sent, + io.pop_x_init_data, io.pop_x_init_dep, - io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id))) + io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id)), + init_tile_id_) } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { @@ -625,23 +630,23 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid - //TODO trackerList(i).io.x_init_data_dep <> x_init_data_dep_arr(trackerList(i).io.init_tile_id) - trackerList(i).io.x_init_data_dep.bits <> MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until NTILES).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) + trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until NTILES).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until NTILES).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) } for( j <- 0 until NTILES ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data + val x_init_data_dep = x_init_data_dep_list(j).io.deq init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid init_arb.io.in(j).bits.xact_init := x_init.bits init_arb.io.in(j).bits.tile_id := UFix(j) - val pop_x_inits = trackerList.map(_.io.pop_x_init && init_arb.io.out.bits.tile_id === UFix(j)) + val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) val do_pop = foldR(pop_x_inits)(_||_) x_init_data_dep_list(j).io.enq.valid := do_pop x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) x_init.ready := (abort_state_arr(j) === s_abort_complete) || do_pop - x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data && init_arb.io.out.bits.tile_id === UFix(j)))(_||_) - x_init_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) + x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) + x_init_data_dep.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) } alloc_arb.io.out.ready := init_arb.io.out.valid From 35ffb80911ec380695fe1629ff695672f00cb33c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 6 Mar 2012 17:33:11 -0800 Subject: [PATCH 047/374] unified coherence trait functions --- uncore/coherence.scala | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index a04b7096..bda67671 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -126,7 +126,15 @@ trait ThreeStateIncoherence extends CoherencePolicy { val (read, write) = cpuCmdToRW(cmd) Mux(write, tileDirty, state) } - + def newTransactionOnMiss(cmd: Bits, state: UFix): UFix = X_INIT_READ_EXCLUSIVE + def newStateOnTransactionRep(cmd: Bits, incoming: TransactionReply, outstanding: TransactionInit): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, tileDirty, tileClean) + } + def needsSecondaryXact(cmd: Bits, outstanding: TransactionInit): Bool = Bool(false) + def newStateOnProbeReq(incoming: ProbeRequest, state: UFix): Bits = state + def probeReplyHasData (reply: ProbeReply): Bool = Bool(false) + def transactionInitHasData (init: TransactionInit): Bool = (init.t_type != X_INIT_WRITE_UNCACHED) } trait FourStateCoherence extends CoherencePolicy { @@ -162,7 +170,7 @@ trait FourStateCoherence extends CoherencePolicy { val (read, write) = cpuCmdToRW(cmd) Mux(write, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) } - def newStateOnTransactionRep(incoming: TransactionReply, outstanding: TransactionInit): UFix = { + def newStateOnTransactionRep(cmd: Bits, incoming: TransactionReply, outstanding: TransactionInit): UFix = { MuxLookup(incoming.t_type, tileInvalid, Array( X_REP_READ_SHARED -> tileShared, X_REP_READ_EXCLUSIVE -> Mux(outstanding.t_type === X_INIT_READ_EXCLUSIVE, tileExclusiveDirty, tileExclusiveClean), From 792960087434d227f164cb63a65579ecfb875a80 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 7 Mar 2012 01:26:35 -0800 Subject: [PATCH 048/374] change D$ to use FourStateCoherence protocol instead of ThreeStateIncoherence. --- uncore/coherence.scala | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index bda67671..2b34b8ae 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -121,17 +121,18 @@ trait ThreeStateIncoherence extends CoherencePolicy { Mux(write, tileDirty, Mux(read, Mux(state === tileDirty, tileDirty, tileClean), state)) } def newStateOnHit(cmd: Bits, state: UFix): UFix = newState(cmd, state) - def newStateOnPrimaryMiss(cmd: Bits): UFix = newState(cmd, tileInvalid) - def newStateOnSecondaryMiss(cmd: Bits, state: UFix): UFix = { + def newTransactionOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, tileDirty, state) + Mux(write, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) } - def newTransactionOnMiss(cmd: Bits, state: UFix): UFix = X_INIT_READ_EXCLUSIVE - def newStateOnTransactionRep(cmd: Bits, incoming: TransactionReply, outstanding: TransactionInit): UFix = { + def newTransactionOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, tileDirty, tileClean) - } + Mux(write, X_INIT_READ_EXCLUSIVE, outstanding.t_type) + } def needsSecondaryXact(cmd: Bits, outstanding: TransactionInit): Bool = Bool(false) + def newStateOnTransactionRep(incoming: TransactionReply, outstanding: TransactionInit): UFix = { + Mux(outstanding.t_type === X_INIT_READ_EXCLUSIVE, tileDirty, tileClean) + } def newStateOnProbeReq(incoming: ProbeRequest, state: UFix): Bits = state def probeReplyHasData (reply: ProbeReply): Bool = Bool(false) def transactionInitHasData (init: TransactionInit): Bool = (init.t_type != X_INIT_WRITE_UNCACHED) @@ -166,11 +167,20 @@ trait FourStateCoherence extends CoherencePolicy { val (read, write) = cpuCmdToRW(cmd) Mux(write, tileExclusiveDirty, state) } - def newTransactionOnMiss(cmd: Bits, state: UFix): UFix = { + def newTransactionOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) Mux(write, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) } - def newStateOnTransactionRep(cmd: Bits, incoming: TransactionReply, outstanding: TransactionInit): UFix = { + def newTransactionOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, X_INIT_READ_EXCLUSIVE, outstanding.t_type) + } + def needsSecondaryXact(cmd: Bits, outstanding: TransactionInit): Bool = { + val (read, write) = cpuCmdToRW(cmd) + (read && (outstanding.t_type === X_INIT_READ_UNCACHED || outstanding.t_type === X_INIT_WRITE_UNCACHED)) || + (write && (outstanding.t_type != X_INIT_READ_EXCLUSIVE)) + } + def newStateOnTransactionRep(incoming: TransactionReply, outstanding: TransactionInit): UFix = { MuxLookup(incoming.t_type, tileInvalid, Array( X_REP_READ_SHARED -> tileShared, X_REP_READ_EXCLUSIVE -> Mux(outstanding.t_type === X_INIT_READ_EXCLUSIVE, tileExclusiveDirty, tileExclusiveClean), @@ -179,11 +189,6 @@ trait FourStateCoherence extends CoherencePolicy { X_REP_WRITE_UNCACHED -> tileInvalid )) } - def needsSecondaryXact(cmd: Bits, outstanding: TransactionInit): Bool = { - val (read, write) = cpuCmdToRW(cmd) - (read && (outstanding.t_type === X_INIT_READ_UNCACHED || outstanding.t_type === X_INIT_WRITE_UNCACHED)) || - (write && (outstanding.t_type != X_INIT_READ_EXCLUSIVE)) - } def newStateOnProbeReq(incoming: ProbeRequest, state: UFix): Bits = { MuxLookup(incoming.p_type, state, Array( From 1a90e2a1624acbf58ed127c042ffe4847f83792e Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 7 Mar 2012 11:40:49 -0800 Subject: [PATCH 049/374] Broadcast hub bug fixes for load uncached mem req and store uncached xact rep --- uncore/coherence.scala | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 2b34b8ae..843ff2be 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -135,7 +135,7 @@ trait ThreeStateIncoherence extends CoherencePolicy { } def newStateOnProbeReq(incoming: ProbeRequest, state: UFix): Bits = state def probeReplyHasData (reply: ProbeReply): Bool = Bool(false) - def transactionInitHasData (init: TransactionInit): Bool = (init.t_type != X_INIT_WRITE_UNCACHED) + def transactionInitHasData (init: TransactionInit): Bool = (init.t_type === X_INIT_WRITE_UNCACHED) } trait FourStateCoherence extends CoherencePolicy { @@ -204,7 +204,7 @@ trait FourStateCoherence extends CoherencePolicy { reply.p_type === P_REP_COPY_DATA) } def transactionInitHasData (init: TransactionInit): Bool = { - (init.t_type != X_INIT_WRITE_UNCACHED) + (init.t_type === X_INIT_WRITE_UNCACHED) } def transactionReplyHasData (reply: TransactionReply): Bool = { (reply.t_type != X_REP_WRITE_UNCACHED && reply.t_type != X_REP_READ_EXCLUSIVE_ACK) @@ -346,7 +346,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { x_init_data_needs_write := transactionInitHasData(io.alloc_req.bits.xact_init) x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) p_rep_count := UFix(NTILES-1) - p_req_flags := ~( UFix(1) << io.alloc_req.bits.tile_id ) + p_req_flags := ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) @@ -516,12 +516,14 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ // Reply to initial requestor // Forward memory responses from mem to tile or arbitrate to ack val mem_idx = io.mem.resp.bits.tag - val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits, NGLOBAL_XACTS) + val ack_idx = UFix(0)//PriorityEncoder(send_x_rep_ack_arr.toBits, NGLOBAL_XACTS) + //val ack_idx_ = Reg(ack_idx) for( j <- 0 until NTILES ) { val rep = io.tiles(j).xact_rep rep.bits.t_type := UFix(0) rep.bits.tile_xact_id := UFix(0) rep.bits.global_xact_id := UFix(0) + rep.bits.data := io.mem.resp.bits.data rep.valid := Bool(false) when(io.mem.resp.valid) { rep.bits.t_type := getTransactionReplyType(t_type_arr(mem_idx), sh_count_arr(mem_idx)) @@ -534,7 +536,6 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ rep.bits.global_xact_id := ack_idx rep.valid := (UFix(j) === init_tile_id_arr(ack_idx)) && send_x_rep_ack_arr(ack_idx) } - io.tiles(j).xact_rep.bits.data := io.mem.resp.bits.data } sent_x_rep_ack_arr(ack_idx) := !io.mem.resp.valid && send_x_rep_ack_arr(ack_idx) // If there were a ready signal due to e.g. intervening network use: From df6ac34821800d07c9b953b19e5af09a7bc3cd68 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 7 Mar 2012 21:03:44 -0800 Subject: [PATCH 050/374] coherence hub fixes --- uncore/coherence.scala | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 843ff2be..042ee666 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -516,8 +516,7 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ // Reply to initial requestor // Forward memory responses from mem to tile or arbitrate to ack val mem_idx = io.mem.resp.bits.tag - val ack_idx = UFix(0)//PriorityEncoder(send_x_rep_ack_arr.toBits, NGLOBAL_XACTS) - //val ack_idx_ = Reg(ack_idx) + val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) for( j <- 0 until NTILES ) { val rep = io.tiles(j).xact_rep rep.bits.t_type := UFix(0) @@ -534,10 +533,10 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ rep.bits.t_type := getTransactionReplyType(t_type_arr(ack_idx), sh_count_arr(ack_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) rep.bits.global_xact_id := ack_idx - rep.valid := (UFix(j) === init_tile_id_arr(ack_idx)) && send_x_rep_ack_arr(ack_idx) + rep.valid := (UFix(j) === init_tile_id_arr(ack_idx)) && send_x_rep_ack_arr.toBits.orR } } - sent_x_rep_ack_arr(ack_idx) := !io.mem.resp.valid && send_x_rep_ack_arr(ack_idx) + sent_x_rep_ack_arr(ack_idx) := !io.mem.resp.valid // If there were a ready signal due to e.g. intervening network use: //io.mem.resp.ready := io.tiles(init_tile_id_arr.read(mem_idx)).xact_rep.ready @@ -592,13 +591,13 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ val x_init_data = io.tiles(j).xact_init_data val x_abort = io.tiles(j).xact_abort val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) - val conflicts = Bits(width = NGLOBAL_XACTS) + val conflicts = Vec(NGLOBAL_XACTS) { Wire() { Bool() } } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(UFix(i), t.busy && x_init.valid && coherenceConflict(t.addr, x_init.bits.address)) + conflicts(i) := t.busy && x_init.valid && coherenceConflict(t.addr, x_init.bits.address) } x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id - want_to_abort_arr(j) := conflicts.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && transactionInitHasData(x_init.bits)) + want_to_abort_arr(j) := conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && transactionInitHasData(x_init.bits)) x_abort.valid := Bool(false) switch(abort_state_arr(j)) { From 3fc94d627d8f6426433418dccb636d245e1dc204 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 8 Mar 2012 11:36:10 -0800 Subject: [PATCH 051/374] Fixed dependency queue bug in Broadcast Hub --- uncore/coherence.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 042ee666..14504062 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -638,7 +638,7 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ for( i <- 0 until NGLOBAL_XACTS ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready - trackerList(i).io.alloc_req.bits <> init_arb.io.out.bits + trackerList(i).io.alloc_req.bits := init_arb.io.out.bits trackerList(i).io.alloc_req.valid := init_arb.io.out.valid trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits @@ -655,7 +655,7 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ init_arb.io.in(j).bits.tile_id := UFix(j) val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) val do_pop = foldR(pop_x_inits)(_||_) - x_init_data_dep_list(j).io.enq.valid := do_pop + x_init_data_dep_list(j).io.enq.valid := do_pop && transactionInitHasData(x_init.bits) && (abort_state_arr(j) === s_idle) x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) x_init.ready := (abort_state_arr(j) === s_abort_complete) || do_pop x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) From ca5caf898c6a26b5074685a4abfdde65d125c8b5 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 8 Mar 2012 16:39:05 -0800 Subject: [PATCH 052/374] Hub addr comparison bug fix --- uncore/coherence.scala | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 14504062..e3827c72 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -42,7 +42,7 @@ class TrackerDependency extends Bundle { class TransactionInit extends Bundle { val t_type = Bits(width = X_INIT_TYPE_BITS) val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) - val address = UFix(width = PADDR_BITS) + val address = UFix(width = PADDR_BITS - OFFSET_BITS) } class TransactionInitData extends MemData @@ -54,7 +54,7 @@ class TransactionAbort extends Bundle { class ProbeRequest extends Bundle { val p_type = Bits(width = P_REQ_TYPE_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) - val address = Bits(width = PADDR_BITS) + val address = Bits(width = PADDR_BITS - OFFSET_BITS) } class ProbeReply extends Bundle { @@ -230,7 +230,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { val mem_req_lock = Bool(OUTPUT) val probe_req = (new ioDecoupled) { new ProbeRequest } val busy = Bool(OUTPUT) - val addr = Bits(PADDR_BITS, OUTPUT) + val addr = Bits(PADDR_BITS - OFFSET_BITS, OUTPUT) val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) @@ -449,9 +449,8 @@ class CoherenceHubNull extends CoherenceHub { class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ - def coherenceConflict(addr1: Bits, addr2: Bits): Bool = { - addr1(PADDR_BITS-1, OFFSET_BITS) === addr2(PADDR_BITS-1, OFFSET_BITS) - } + def coherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) + def getTransactionReplyType(t_type: UFix, count: UFix): Bits = { MuxLookup(t_type, X_REP_READ_UNCACHED, Array( X_INIT_READ_SHARED -> Mux(count > UFix(0), X_REP_READ_SHARED, X_REP_READ_EXCLUSIVE), @@ -464,7 +463,7 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } + val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } From 2df5c19f13adf1ae67a9955122c3d6de1f4b51d0 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 8 Mar 2012 18:07:44 -0800 Subject: [PATCH 053/374] Added require_ack field to TransactionReply bundle --- uncore/coherence.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index e3827c72..a2c6ba5f 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -68,6 +68,7 @@ class TransactionReply extends MemData { val t_type = Bits(width = X_REP_TYPE_BITS) val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) + val require_ack = Bool() } class TransactionFinish extends Bundle { @@ -440,6 +441,7 @@ class CoherenceHubNull extends CoherenceHub { x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.bits.data + x_rep.bits.require_ack := Bool(true) x_rep.valid := io.mem.resp.valid || x_init.valid && is_write io.tiles(0).xact_abort.valid := Bool(false) @@ -522,6 +524,7 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ rep.bits.tile_xact_id := UFix(0) rep.bits.global_xact_id := UFix(0) rep.bits.data := io.mem.resp.bits.data + rep.bits.require_ack := Bool(true) rep.valid := Bool(false) when(io.mem.resp.valid) { rep.bits.t_type := getTransactionReplyType(t_type_arr(mem_idx), sh_count_arr(mem_idx)) From 2607153b6785875ead46253bc513f86578c0fa53 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 9 Mar 2012 02:08:55 -0800 Subject: [PATCH 054/374] Merge branch 'master' of github.com:ucb-bar/riscv-rocket From 2014db41bd0e826ee5469b8149c6284896a737d2 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 9 Mar 2012 11:04:58 -0800 Subject: [PATCH 055/374] Special cased NTILES == 1 due to log2up revision --- uncore/coherence.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index a2c6ba5f..275e427b 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -297,7 +297,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { val t_type_ = Reg{ Bits() } val init_tile_id_ = Reg{ Bits() } val tile_xact_id_ = Reg{ Bits() } - val p_rep_count = Reg(resetVal = UFix(0, width = log2up(NTILES))) + val p_rep_count = if (NTILES == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2up(NTILES))) val p_req_flags = Reg(resetVal = Bits(0, width = NTILES)) val p_rep_tile_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) @@ -346,7 +346,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := transactionInitHasData(io.alloc_req.bits.xact_init) x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) - p_rep_count := UFix(NTILES-1) + if(NTILES > 1) p_rep_count := UFix(NTILES-1) p_req_flags := ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) @@ -366,7 +366,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { when(io.p_rep_cnt_dec.orR) { val p_rep_count_next = p_rep_count - PopCount(io.p_rep_cnt_dec) io.pop_p_rep := io.p_rep_cnt_dec - p_rep_count := p_rep_count_next + if(NTILES > 1) p_rep_count := p_rep_count_next when(p_rep_count === UFix(0)) { io.pop_p_rep := Bool(true) state := s_mem From d777f1cb24cb951ece5962f79677552a7e899758 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sat, 10 Mar 2012 15:18:57 -0800 Subject: [PATCH 056/374] fix null hub store ack bug --- uncore/coherence.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 275e427b..41959915 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -124,7 +124,7 @@ trait ThreeStateIncoherence extends CoherencePolicy { def newStateOnHit(cmd: Bits, state: UFix): UFix = newState(cmd, state) def newTransactionOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) + Mux(write || cmd === M_PFW, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) } def newTransactionOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -170,7 +170,7 @@ trait FourStateCoherence extends CoherencePolicy { } def newTransactionOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) + Mux(write || cmd === M_PFW, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) } def newTransactionOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -442,7 +442,7 @@ class CoherenceHubNull extends CoherenceHub { x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.bits.data x_rep.bits.require_ack := Bool(true) - x_rep.valid := io.mem.resp.valid || x_init.valid && is_write + x_rep.valid := io.mem.resp.valid || x_init.valid && is_write && io.mem.req_cmd.ready io.tiles(0).xact_abort.valid := Bool(false) io.tiles(0).xact_finish.ready := Bool(true) From cb5ce3fe7310c6ae7686665f03bc40a8a8958932 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 11 Mar 2012 14:17:27 -0700 Subject: [PATCH 057/374] More broadcast hub bugfixes --- uncore/coherence.scala | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 41959915..f128036a 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -276,17 +276,17 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { when(req_data.ready && req_data.valid) { pop_data := UFix(1) << tile_id mem_cnt := mem_cnt_next - } - when(mem_cnt_next === UFix(0)) { - pop_dep := UFix(1) << tile_id - trigger := Bool(false) + when(mem_cnt_next === UFix(0)) { + pop_dep := UFix(1) << tile_id + trigger := Bool(false) + } } } def doMemReqRead(req_cmd: ioDecoupled[MemReqCmd], trigger: Bool) { req_cmd.valid := Bool(true) req_cmd.bits.rw := Bool(false) - when(req_cmd.ready ) { + when(req_cmd.ready) { trigger := Bool(false) } } @@ -535,10 +535,11 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ rep.bits.t_type := getTransactionReplyType(t_type_arr(ack_idx), sh_count_arr(ack_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) rep.bits.global_xact_id := ack_idx - rep.valid := (UFix(j) === init_tile_id_arr(ack_idx)) && send_x_rep_ack_arr.toBits.orR + val do_send_ack = (UFix(j) === init_tile_id_arr(ack_idx)) && send_x_rep_ack_arr.toBits.orR + rep.valid := do_send_ack + sent_x_rep_ack_arr(ack_idx) := do_send_ack } } - sent_x_rep_ack_arr(ack_idx) := !io.mem.resp.valid // If there were a ready signal due to e.g. intervening network use: //io.mem.resp.ready := io.tiles(init_tile_id_arr.read(mem_idx)).xact_rep.ready From 95f880da704542ea7d346cbb275610ac8cc30ff7 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 11 Mar 2012 17:13:01 -0700 Subject: [PATCH 058/374] fixed abort bug --- uncore/coherence.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index f128036a..3db3ac46 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -600,7 +600,7 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ conflicts(i) := t.busy && x_init.valid && coherenceConflict(t.addr, x_init.bits.address) } x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id - want_to_abort_arr(j) := conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && transactionInitHasData(x_init.bits)) + want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && transactionInitHasData(x_init.bits))) x_abort.valid := Bool(false) switch(abort_state_arr(j)) { From 23c822a82e854a70ebc7bd1303e89bd532604600 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 12 Mar 2012 10:38:37 -0700 Subject: [PATCH 059/374] fix hit logic for amos --- uncore/coherence.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 3db3ac46..653ab6dc 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -147,8 +147,8 @@ trait FourStateCoherence extends CoherencePolicy { def isHit ( cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) - ((read && ( state === tileShared || state === tileExclusiveClean || state === tileExclusiveDirty)) || - (write && (state === tileExclusiveClean || state === tileExclusiveDirty))) + Mux(write, (state === tileExclusiveClean || state === tileExclusiveDirty), + (state === tileShared || state === tileExclusiveClean || state === tileExclusiveDirty)) } //TODO: do we need isPresent() for determining that a line needs to be From 1b733e7cf07c9e8abd95358198bf06d7ade92f48 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Tue, 13 Mar 2012 12:34:39 -0700 Subject: [PATCH 060/374] Merge branch 'master' of github.com:ucb-bar/riscv-rocket From 1258f31825c7a3e0787784de4fbfe0c86704dfdf Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 13 Mar 2012 16:43:35 -0700 Subject: [PATCH 061/374] add probe unit --- uncore/coherence.scala | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 653ab6dc..de158417 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -135,6 +135,12 @@ trait ThreeStateIncoherence extends CoherencePolicy { Mux(outstanding.t_type === X_INIT_READ_EXCLUSIVE, tileDirty, tileClean) } def newStateOnProbeReq(incoming: ProbeRequest, state: UFix): Bits = state + def newProbeReply (incoming: ProbeRequest, has_data: Bool): ProbeReply = { + val reply = Wire() { new ProbeReply() } + reply.p_type := P_REP_INVALIDATE_ACK + reply.global_xact_id := UFix(0) + reply + } def probeReplyHasData (reply: ProbeReply): Bool = Bool(false) def transactionInitHasData (init: TransactionInit): Bool = (init.t_type === X_INIT_WRITE_UNCACHED) } @@ -199,6 +205,22 @@ trait FourStateCoherence extends CoherencePolicy { )) } + def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { + val reply = Wire() { new ProbeReply() } + val with_data = MuxLookup(incoming.p_type, state, Array( + probeInvalidate -> P_REP_INVALIDATE_DATA, + probeDowngrade -> P_REP_DOWNGRADE_DATA, + probeCopy -> P_REP_COPY_DATA + )) + val without_data = MuxLookup(incoming.p_type, state, Array( + probeInvalidate -> P_REP_INVALIDATE_ACK, + probeDowngrade -> P_REP_DOWNGRADE_ACK, + probeCopy -> P_REP_COPY_ACK + )) + reply.p_type := Mux(needsWriteback(state), with_data, without_data) + reply.global_xact_id := incoming.global_xact_id + reply + } def probeReplyHasData (reply: ProbeReply): Bool = { (reply.p_type === P_REP_INVALIDATE_DATA || reply.p_type === P_REP_DOWNGRADE_DATA || @@ -446,6 +468,9 @@ class CoherenceHubNull extends CoherenceHub { io.tiles(0).xact_abort.valid := Bool(false) io.tiles(0).xact_finish.ready := Bool(true) + io.tiles(0).probe_req.valid := Bool(false) + io.tiles(0).probe_rep.ready := Bool(true) + io.tiles(0).probe_rep_data.ready := Bool(true) } From 53d69d3006b5e52b1b881fcd135eaa8240e734af Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 13 Mar 2012 17:12:01 -0700 Subject: [PATCH 062/374] parameterize broadcast hub by # of tiles --- uncore/coherence.scala | 94 +++++++++++++++++++++--------------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index de158417..ae9321b1 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -234,14 +234,14 @@ trait FourStateCoherence extends CoherencePolicy { } } -class XactTracker(id: Int) extends Component with FourStateCoherence { +class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherence { val io = new Bundle { val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip val p_data = (new ioPipe) { new TrackerProbeData } val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) - val p_rep_cnt_dec = Bits(NTILES, INPUT) - val p_req_cnt_inc = Bits(NTILES, INPUT) + val p_rep_cnt_dec = Bits(ntiles, INPUT) + val p_req_cnt_inc = Bits(ntiles, INPUT) val p_rep_data = (new ioPipe) { new ProbeReplyData }.flip val x_init_data = (new ioPipe) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) @@ -259,13 +259,13 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) val t_type = Bits(X_INIT_TYPE_BITS, OUTPUT) - val push_p_req = Bits(NTILES, OUTPUT) - val pop_p_rep = Bits(NTILES, OUTPUT) - val pop_p_rep_data = Bits(NTILES, OUTPUT) - val pop_p_rep_dep = Bits(NTILES, OUTPUT) - val pop_x_init = Bits(NTILES, OUTPUT) - val pop_x_init_data = Bits(NTILES, OUTPUT) - val pop_x_init_dep = Bits(NTILES, OUTPUT) + val push_p_req = Bits(ntiles, OUTPUT) + val pop_p_rep = Bits(ntiles, OUTPUT) + val pop_p_rep_data = Bits(ntiles, OUTPUT) + val pop_p_rep_dep = Bits(ntiles, OUTPUT) + val pop_x_init = Bits(ntiles, OUTPUT) + val pop_x_init_data = Bits(ntiles, OUTPUT) + val pop_x_init_dep = Bits(ntiles, OUTPUT) val send_x_rep_ack = Bool(OUTPUT) } @@ -319,8 +319,8 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { val t_type_ = Reg{ Bits() } val init_tile_id_ = Reg{ Bits() } val tile_xact_id_ = Reg{ Bits() } - val p_rep_count = if (NTILES == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2up(NTILES))) - val p_req_flags = Reg(resetVal = Bits(0, width = NTILES)) + val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2up(ntiles))) + val p_req_flags = Reg(resetVal = Bits(0, width = ntiles)) val p_rep_tile_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) val x_init_data_needs_write = Reg(resetVal = Bool(false)) @@ -336,7 +336,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { io.init_tile_id := init_tile_id_ io.p_rep_tile_id := p_rep_tile_id_ io.tile_xact_id := tile_xact_id_ - io.sharer_count := UFix(NTILES) // TODO: Broadcast only + io.sharer_count := UFix(ntiles) // TODO: Broadcast only io.t_type := t_type_ io.mem_req_cmd.valid := Bool(false) @@ -350,13 +350,13 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { io.probe_req.bits.p_type := sendProbeReqType(t_type_, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) io.probe_req.bits.address := addr_ - io.push_p_req := Bits(0, width = NTILES) - io.pop_p_rep := Bits(0, width = NTILES) - io.pop_p_rep_data := Bits(0, width = NTILES) - io.pop_p_rep_dep := Bits(0, width = NTILES) - io.pop_x_init := Bits(0, width = NTILES) - io.pop_x_init_data := Bits(0, width = NTILES) - io.pop_x_init_dep := Bits(0, width = NTILES) + io.push_p_req := Bits(0, width = ntiles) + io.pop_p_rep := Bits(0, width = ntiles) + io.pop_p_rep_data := Bits(0, width = ntiles) + io.pop_p_rep_dep := Bits(0, width = ntiles) + io.pop_x_init := Bits(0, width = ntiles) + io.pop_x_init_data := Bits(0, width = ntiles) + io.pop_x_init_dep := Bits(0, width = ntiles) io.send_x_rep_ack := Bool(false) switch (state) { @@ -368,7 +368,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := transactionInitHasData(io.alloc_req.bits.xact_init) x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) - if(NTILES > 1) p_rep_count := UFix(NTILES-1) + if(ntiles > 1) p_rep_count := UFix(ntiles-1) p_req_flags := ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) @@ -388,7 +388,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { when(io.p_rep_cnt_dec.orR) { val p_rep_count_next = p_rep_count - PopCount(io.p_rep_cnt_dec) io.pop_p_rep := io.p_rep_cnt_dec - if(NTILES > 1) p_rep_count := p_rep_count_next + if(ntiles > 1) p_rep_count := p_rep_count_next when(p_rep_count === UFix(0)) { io.pop_p_rep := Bool(true) state := s_mem @@ -440,14 +440,14 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { } } -abstract class CoherenceHub extends Component with CoherencePolicy { +abstract class CoherenceHub(ntiles: Int) extends Component with CoherencePolicy { val io = new Bundle { - val tiles = Vec(NTILES) { new ioTileLink() }.flip + val tiles = Vec(ntiles) { new ioTileLink() }.flip val mem = new ioMem } } -class CoherenceHubNull extends CoherenceHub { +class CoherenceHubNull extends CoherenceHub(1) { val x_init = io.tiles(0).xact_init val is_write = x_init.bits.t_type === X_INIT_WRITE_UNCACHED @@ -474,7 +474,7 @@ class CoherenceHubNull extends CoherenceHub { } -class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ +class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourStateCoherence{ def coherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) @@ -487,7 +487,7 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ )) } - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _)) val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } @@ -498,8 +498,8 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(NTILES){ Wire(){Bool()} } } - val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(NTILES){ Wire(){Bool()} } } + val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } + val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bits(width = TILE_ID_BITS)} } val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } @@ -523,17 +523,17 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ sent_x_rep_ack_arr(i) := Bool(false) p_data_tile_id_arr(i) := Bits(0, width = TILE_ID_BITS) p_data_valid_arr(i) := Bool(false) - for( j <- 0 until NTILES) { + for( j <- 0 until ntiles) { p_rep_cnt_dec_arr(i)(j) := Bool(false) p_req_cnt_inc_arr(i)(j) := Bool(false) } } - val p_rep_data_dep_list = List.fill(NTILES)((new queue(NGLOBAL_XACTS, true)){new TrackerDependency}) // depth must >= NPRIMARY - val x_init_data_dep_list = List.fill(NTILES)((new queue(NGLOBAL_XACTS, true)){new TrackerDependency}) // depth should >= NPRIMARY + val p_rep_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS, true)){new TrackerDependency}) // depth must >= NPRIMARY + val x_init_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS, true)){new TrackerDependency}) // depth should >= NPRIMARY // Free finished transactions - for( j <- 0 until NTILES ) { + for( j <- 0 until ntiles ) { val finish = io.tiles(j).xact_finish do_free_arr(finish.bits.global_xact_id) := finish.valid finish.ready := Bool(true) @@ -543,7 +543,7 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ // Forward memory responses from mem to tile or arbitrate to ack val mem_idx = io.mem.resp.bits.tag val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) - for( j <- 0 until NTILES ) { + for( j <- 0 until ntiles ) { val rep = io.tiles(j).xact_rep rep.bits.t_type := UFix(0) rep.bits.tile_xact_id := UFix(0) @@ -583,7 +583,7 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ io.mem.req_data <> mem_req_data_arb.io.out // Handle probe replies, which may or may not have data - for( j <- 0 until NTILES ) { + for( j <- 0 until ntiles ) { val p_rep = io.tiles(j).probe_rep val p_rep_data = io.tiles(j).probe_rep_data val idx = p_rep.bits.global_xact_id @@ -601,10 +601,10 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits - trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until NTILES).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) - trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until NTILES).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) + trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) + trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) - for( j <- 0 until NTILES) { + for( j <- 0 until ntiles) { val p_rep = io.tiles(j).probe_rep p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) } @@ -612,9 +612,9 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ // Nack conflicting transaction init attempts val s_idle :: s_abort_drain :: s_abort_send :: s_abort_complete :: Nil = Enum(4){ UFix() } - val abort_state_arr = Vec(NTILES) { Reg(resetVal = s_idle) } - val want_to_abort_arr = Vec(NTILES) { Wire() { Bool()} } - for( j <- 0 until NTILES ) { + val abort_state_arr = Vec(ntiles) { Reg(resetVal = s_idle) } + val want_to_abort_arr = Vec(ntiles) { Wire() { Bool()} } + for( j <- 0 until ntiles ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_abort = io.tiles(j).xact_abort @@ -662,7 +662,7 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ // Only one allocation per cycle // Init requests may or may not have data val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } - val init_arb = (new Arbiter(NTILES)) { new TrackerAllocReq() } + val init_arb = (new Arbiter(ntiles)) { new TrackerAllocReq() } for( i <- 0 until NGLOBAL_XACTS ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready @@ -671,10 +671,10 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid - trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until NTILES).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) - trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until NTILES).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) + trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) + trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) } - for( j <- 0 until NTILES ) { + for( j <- 0 until ntiles ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_init_data_dep = x_init_data_dep_list(j).io.deq @@ -694,8 +694,8 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ // Handle probe request generation // Must arbitrate for each request port - val p_req_arb_arr = List.fill(NTILES)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) - for( j <- 0 until NTILES ) { + val p_req_arb_arr = List.fill(ntiles)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) + for( j <- 0 until ntiles ) { for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits From 53cd543d3f6ad91c44262225431b1558836256a0 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 13 Mar 2012 19:10:54 -0700 Subject: [PATCH 063/374] fix minor coherence bugs --- uncore/coherence.scala | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index ae9321b1..fc32b6eb 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -207,12 +207,12 @@ trait FourStateCoherence extends CoherencePolicy { def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { val reply = Wire() { new ProbeReply() } - val with_data = MuxLookup(incoming.p_type, state, Array( + val with_data = MuxLookup(incoming.p_type, P_REP_INVALIDATE_DATA, Array( probeInvalidate -> P_REP_INVALIDATE_DATA, probeDowngrade -> P_REP_DOWNGRADE_DATA, probeCopy -> P_REP_COPY_DATA )) - val without_data = MuxLookup(incoming.p_type, state, Array( + val without_data = MuxLookup(incoming.p_type, P_REP_INVALIDATE_ACK, Array( probeInvalidate -> P_REP_INVALIDATE_ACK, probeDowngrade -> P_REP_DOWNGRADE_ACK, probeCopy -> P_REP_COPY_ACK @@ -369,12 +369,13 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc x_init_data_needs_write := transactionInitHasData(io.alloc_req.bits.xact_init) x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) if(ntiles > 1) p_rep_count := UFix(ntiles-1) - p_req_flags := ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only + val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only + p_req_flags := p_req_initial_flags mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id - state := Mux(p_req_flags.orR, s_probe, s_mem) + state := Mux(p_req_initial_flags.orR, s_probe, s_mem) } } is(s_probe) { @@ -389,7 +390,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc val p_rep_count_next = p_rep_count - PopCount(io.p_rep_cnt_dec) io.pop_p_rep := io.p_rep_cnt_dec if(ntiles > 1) p_rep_count := p_rep_count_next - when(p_rep_count === UFix(0)) { + when(p_rep_count === UFix(1)) { io.pop_p_rep := Bool(true) state := s_mem } @@ -529,8 +530,8 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS } } - val p_rep_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS, true)){new TrackerDependency}) // depth must >= NPRIMARY - val x_init_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS, true)){new TrackerDependency}) // depth should >= NPRIMARY + val p_rep_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY + val x_init_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY // Free finished transactions for( j <- 0 until ntiles ) { @@ -589,7 +590,7 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS val idx = p_rep.bits.global_xact_id val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) val do_pop = foldR(pop_p_reps)(_ || _) - p_rep.ready := do_pop + p_rep.ready := Bool(true) p_rep_data_dep_list(j).io.enq.valid := do_pop p_rep_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_p_reps) p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) From 77c405ffa17d1121439a59d32d6fa38f876d9856 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 14 Mar 2012 06:13:16 -0700 Subject: [PATCH 064/374] use broadcast hub and coherent HTIF --- uncore/coherence.scala | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index fc32b6eb..7cf36e8a 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -237,7 +237,7 @@ trait FourStateCoherence extends CoherencePolicy { class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherence { val io = new Bundle { val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip - val p_data = (new ioPipe) { new TrackerProbeData } + val p_data = (new ioPipe) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) val p_rep_cnt_dec = Bits(ntiles, INPUT) @@ -387,10 +387,10 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc p_req_flags := p_req_flags & ~io.p_req_cnt_inc // unflag sent reqs } when(io.p_rep_cnt_dec.orR) { - val p_rep_count_next = p_rep_count - PopCount(io.p_rep_cnt_dec) + val dec = PopCount(io.p_rep_cnt_dec) io.pop_p_rep := io.p_rep_cnt_dec - if(ntiles > 1) p_rep_count := p_rep_count_next - when(p_rep_count === UFix(1)) { + if(ntiles > 1) p_rep_count := p_rep_count - dec + when(p_rep_count === dec) { io.pop_p_rep := Bool(true) state := s_mem } @@ -536,7 +536,9 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS // Free finished transactions for( j <- 0 until ntiles ) { val finish = io.tiles(j).xact_finish - do_free_arr(finish.bits.global_xact_id) := finish.valid + when (finish.valid) { + do_free_arr(finish.bits.global_xact_id) := Bool(true) + } finish.ready := Bool(true) } @@ -552,18 +554,19 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS rep.bits.data := io.mem.resp.bits.data rep.bits.require_ack := Bool(true) rep.valid := Bool(false) - when(io.mem.resp.valid) { + when(io.mem.resp.valid && (UFix(j) === init_tile_id_arr(mem_idx))) { rep.bits.t_type := getTransactionReplyType(t_type_arr(mem_idx), sh_count_arr(mem_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) rep.bits.global_xact_id := mem_idx - rep.valid := (UFix(j) === init_tile_id_arr(mem_idx)) + rep.valid := Bool(true) } . otherwise { rep.bits.t_type := getTransactionReplyType(t_type_arr(ack_idx), sh_count_arr(ack_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) rep.bits.global_xact_id := ack_idx - val do_send_ack = (UFix(j) === init_tile_id_arr(ack_idx)) && send_x_rep_ack_arr.toBits.orR - rep.valid := do_send_ack - sent_x_rep_ack_arr(ack_idx) := do_send_ack + when (UFix(j) === init_tile_id_arr(ack_idx)) { + rep.valid := send_x_rep_ack_arr.toBits.orR + sent_x_rep_ack_arr(ack_idx) := Bool(true) + } } } // If there were a ready signal due to e.g. intervening network use: @@ -594,8 +597,10 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS p_rep_data_dep_list(j).io.enq.valid := do_pop p_rep_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_p_reps) p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) - p_data_valid_arr(idx) := p_rep.valid && probeReplyHasData(p_rep.bits) - p_data_tile_id_arr(idx) := UFix(j) + when (p_rep.valid) { + p_data_valid_arr(idx) := probeReplyHasData(p_rep.bits) + p_data_tile_id_arr(idx) := UFix(j) + } p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) } for( i <- 0 until NGLOBAL_XACTS ) { From 3129040bda033ae34deffc14f7c4f8f3e0b58f8d Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 15 Mar 2012 18:36:07 -0700 Subject: [PATCH 065/374] use divided clk for htif. UDPATE YOUR FESVR by default, we now load programs via a backdoor, because otherwise it takes too long to simulate. --- uncore/slowio.scala | 49 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 uncore/slowio.scala diff --git a/uncore/slowio.scala b/uncore/slowio.scala new file mode 100644 index 00000000..c1535044 --- /dev/null +++ b/uncore/slowio.scala @@ -0,0 +1,49 @@ +package rocket + +import Chisel._ +import Constants._ + +class slowIO[T <: Data](divisor: Int, hold_cycles: Int)(data: => T) extends Component +{ + val io = new Bundle { + val out_fast = new ioDecoupled()(data).flip + val out_slow = new ioDecoupled()(data) + + val in_fast = new ioDecoupled()(data) + val in_slow = new ioDecoupled()(data).flip + + val clk_slow = Bool(OUTPUT) + } + + require((divisor & (divisor-1)) == 0) + require(hold_cycles < divisor/2 && hold_cycles >= 2) + + val cnt = Reg() { UFix(width = log2up(divisor)) } + cnt := cnt + UFix(1) + val out_en = cnt === UFix(divisor/2+hold_cycles-1) // rising edge + hold time + val in_en = cnt === UFix(divisor/2-1) // rising edge + + val in_slow_rdy = Reg(resetVal = Bool(false)) + val out_slow_val = Reg(resetVal = Bool(false)) + val out_slow_bits = Reg() { data } + + val fromhost_q = new queue(1)(data) + fromhost_q.io.enq.valid := in_en && io.in_slow.valid && in_slow_rdy + fromhost_q.io.enq.bits := io.in_slow.bits + fromhost_q.io.deq <> io.in_fast + + val tohost_q = new queue(1)(data) + tohost_q.io.enq <> io.out_fast + tohost_q.io.deq.ready := in_en && io.out_slow.ready && out_slow_val + + when (out_en) { + in_slow_rdy := fromhost_q.io.enq.ready + out_slow_val := tohost_q.io.deq.valid + out_slow_bits := tohost_q.io.deq.bits + } + + io.in_slow.ready := in_slow_rdy + io.out_slow.valid := out_slow_val + io.out_slow.bits := out_slow_bits + io.clk_slow := cnt(log2up(divisor)-1).toBool +} From e38114e4b0f87777a98d3e8e85f8870d3e6bf269 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 16 Mar 2012 01:24:07 -0700 Subject: [PATCH 066/374] fix coherence bug popping wrong store dependence queue --- uncore/coherence.scala | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 7cf36e8a..9160a84b 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -295,12 +295,14 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc when(req_cmd.ready && req_cmd.valid) { cmd_sent := Bool(true) } - when(req_data.ready && req_data.valid) { - pop_data := UFix(1) << tile_id - mem_cnt := mem_cnt_next - when(mem_cnt_next === UFix(0)) { - pop_dep := UFix(1) << tile_id - trigger := Bool(false) + when(req_data.ready && at_front_of_dep_queue) { + pop_data := UFix(1) << tile_id + when (data.valid) { + mem_cnt := mem_cnt_next + when(mem_cnt_next === UFix(0)) { + pop_dep := UFix(1) << tile_id + trigger := Bool(false) + } } } } @@ -391,7 +393,6 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc io.pop_p_rep := io.p_rep_cnt_dec if(ntiles > 1) p_rep_count := p_rep_count - dec when(p_rep_count === dec) { - io.pop_p_rep := Bool(true) state := s_mem } } From 5f69c5a764689ee6c85712563100d6b03ffaf5b0 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 20 Mar 2012 00:18:32 -0700 Subject: [PATCH 067/374] fix bug in coherence hub, specifically in abort handling logic --- uncore/coherence.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 9160a84b..9cd8fc7c 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -648,9 +648,9 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS is(s_abort_drain) { // raises x_init_data.ready below when(x_init_data.valid) { abort_cnt := abort_cnt + UFix(1) - } - when(abort_cnt === ~UFix(0, width = log2up(REFILL_CYCLES))) { - abort_state_arr(j) := s_abort_send + when(abort_cnt === ~UFix(0, width = log2up(REFILL_CYCLES))) { + abort_state_arr(j) := s_abort_send + } } } is(s_abort_send) { // nothing is dequeued for now From a7ebea13fcab7fc79b56ad402655531ed8c4650a Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sun, 25 Mar 2012 17:03:58 -0700 Subject: [PATCH 068/374] add mem serdes unit --- uncore/memserdes.scala | 89 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 uncore/memserdes.scala diff --git a/uncore/memserdes.scala b/uncore/memserdes.scala new file mode 100644 index 00000000..2305ed69 --- /dev/null +++ b/uncore/memserdes.scala @@ -0,0 +1,89 @@ +package rocket + +import Chisel._ +import Node._ +import Constants._ +import scala.math._ + +class ioMemSerialized(w: Int) extends Bundle +{ + val req = (new ioDecoupled) { Bits(width = w) } + val resp = (new ioPipe) { Bits(width = w) }.flip +} + +class MemSerdes(w: Int) extends Component +{ + val io = new Bundle { + val wide = new ioMem().flip + val narrow = new ioMemSerialized(w) + } + val abits = io.wide.req_cmd.bits.toBits.getWidth + val dbits = io.wide.req_data.bits.toBits.getWidth + val rbits = io.wide.resp.bits.getWidth + + val out_buf = Reg() { Bits() } + val in_buf = Reg() { Bits() } + + val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(5) { UFix() } + val state = Reg(resetVal = s_idle) + val send_cnt = Reg(resetVal = UFix(0, log2up(max(abits, dbits)))) + val data_send_cnt = Reg(resetVal = UFix(0, log2up(MEM_DATA_BITS))) + val adone = io.narrow.req.ready && send_cnt === UFix((abits-1)/w) + val ddone = io.narrow.req.ready && send_cnt === UFix((dbits-1)/w) + + when (state === s_idle) { + when (io.wide.req_cmd.valid) { + state := Mux(io.wide.req_cmd.bits.rw, s_write_addr, s_read_addr) + } + } + when (state === s_read_addr && adone) { + state := s_idle + send_cnt := UFix(0) + } + when (state === s_write_addr && adone) { + state := s_write_idle + send_cnt := UFix(0) + } + when (state === s_write_idle && io.wide.req_data.valid) { + state := s_write_data + } + when (state === s_write_data && ddone) { + data_send_cnt := data_send_cnt + UFix(1) + state := Mux(data_send_cnt === UFix(REFILL_CYCLES-1), s_idle, s_write_idle) + } + + when (io.narrow.req.valid && io.narrow.req.ready) { + send_cnt := Mux(adone, UFix(0), send_cnt + UFix(1)) + out_buf := out_buf >> UFix(w) + } + when (io.wide.req_cmd.valid && io.wide.req_cmd.ready) { + out_buf := io.wide.req_cmd.bits.toBits + } + when (io.wide.req_data.valid && io.wide.req_data.ready) { + out_buf := io.wide.req_data.bits.toBits + } + + io.wide.req_cmd.ready := state === s_idle + io.wide.req_data.ready := state === s_write_idle + io.narrow.req.valid := state === s_read_addr || state === s_write_addr || state === s_write_data + io.narrow.req.bits := out_buf + + val recv_cnt = Reg() { UFix(width = log2up(rbits)) } + val data_recv_cnt = Reg(resetVal = UFix(0, log2up(MEM_DATA_BITS))) + val resp_val = Reg(resetVal = Bool(false)) + + resp_val := Bool(false) + when (io.narrow.resp.valid) { + recv_cnt := recv_cnt + UFix(1) + when (recv_cnt === UFix((rbits-1)/w)) { + recv_cnt := UFix(0) + data_recv_cnt := data_recv_cnt + UFix(1) + resp_val := Bool(true) + } + in_buf := Cat(io.narrow.resp.bits, in_buf(rbits-1,w)) + } + + io.wide.resp.valid := resp_val + io.wide.resp.bits.tag := in_buf(io.wide.resp.bits.tag.width-1,0) + io.wide.resp.bits.data := in_buf >> UFix(io.wide.resp.bits.tag.width) +} From 5a00143035d1053bd30db2daac4460360c470fa1 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sun, 25 Mar 2012 21:45:10 -0700 Subject: [PATCH 069/374] loop host.in to host.out during reset --- uncore/slowio.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/slowio.scala b/uncore/slowio.scala index c1535044..395bdc8d 100644 --- a/uncore/slowio.scala +++ b/uncore/slowio.scala @@ -28,7 +28,7 @@ class slowIO[T <: Data](divisor: Int, hold_cycles: Int)(data: => T) extends Comp val out_slow_bits = Reg() { data } val fromhost_q = new queue(1)(data) - fromhost_q.io.enq.valid := in_en && io.in_slow.valid && in_slow_rdy + fromhost_q.io.enq.valid := in_en && (io.in_slow.valid && in_slow_rdy || reset) fromhost_q.io.enq.bits := io.in_slow.bits fromhost_q.io.deq <> io.in_fast @@ -39,7 +39,7 @@ class slowIO[T <: Data](divisor: Int, hold_cycles: Int)(data: => T) extends Comp when (out_en) { in_slow_rdy := fromhost_q.io.enq.ready out_slow_val := tohost_q.io.deq.valid - out_slow_bits := tohost_q.io.deq.bits + out_slow_bits := Mux(reset, fromhost_q.io.deq.bits, tohost_q.io.deq.bits) } io.in_slow.ready := in_slow_rdy From 4e6302fedc326e2c95549c10839561e0c5d2759f Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sun, 25 Mar 2012 23:03:20 -0700 Subject: [PATCH 070/374] add dessert --- uncore/memserdes.scala | 117 ++++++++++++++++++++++++++++++++--------- 1 file changed, 93 insertions(+), 24 deletions(-) diff --git a/uncore/memserdes.scala b/uncore/memserdes.scala index 2305ed69..f68d36e8 100644 --- a/uncore/memserdes.scala +++ b/uncore/memserdes.scala @@ -27,14 +27,28 @@ class MemSerdes(w: Int) extends Component val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(5) { UFix() } val state = Reg(resetVal = s_idle) val send_cnt = Reg(resetVal = UFix(0, log2up(max(abits, dbits)))) - val data_send_cnt = Reg(resetVal = UFix(0, log2up(MEM_DATA_BITS))) + val data_send_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) val adone = io.narrow.req.ready && send_cnt === UFix((abits-1)/w) val ddone = io.narrow.req.ready && send_cnt === UFix((dbits-1)/w) - when (state === s_idle) { - when (io.wide.req_cmd.valid) { - state := Mux(io.wide.req_cmd.bits.rw, s_write_addr, s_read_addr) - } + when (io.narrow.req.valid && io.narrow.req.ready) { + send_cnt := send_cnt + UFix(1) + out_buf := out_buf >> UFix(w) + } + when (io.wide.req_cmd.valid && io.wide.req_cmd.ready) { + out_buf := io.wide.req_cmd.bits.toBits + } + when (io.wide.req_data.valid && io.wide.req_data.ready) { + out_buf := io.wide.req_data.bits.toBits + } + + io.wide.req_cmd.ready := state === s_idle + io.wide.req_data.ready := state === s_write_idle + io.narrow.req.valid := state === s_read_addr || state === s_write_addr || state === s_write_data + io.narrow.req.bits := out_buf + + when (state === s_idle && io.wide.req_cmd.valid) { + state := Mux(io.wide.req_cmd.bits.rw, s_write_addr, s_read_addr) } when (state === s_read_addr && adone) { state := s_idle @@ -50,26 +64,11 @@ class MemSerdes(w: Int) extends Component when (state === s_write_data && ddone) { data_send_cnt := data_send_cnt + UFix(1) state := Mux(data_send_cnt === UFix(REFILL_CYCLES-1), s_idle, s_write_idle) + send_cnt := UFix(0) } - when (io.narrow.req.valid && io.narrow.req.ready) { - send_cnt := Mux(adone, UFix(0), send_cnt + UFix(1)) - out_buf := out_buf >> UFix(w) - } - when (io.wide.req_cmd.valid && io.wide.req_cmd.ready) { - out_buf := io.wide.req_cmd.bits.toBits - } - when (io.wide.req_data.valid && io.wide.req_data.ready) { - out_buf := io.wide.req_data.bits.toBits - } - - io.wide.req_cmd.ready := state === s_idle - io.wide.req_data.ready := state === s_write_idle - io.narrow.req.valid := state === s_read_addr || state === s_write_addr || state === s_write_data - io.narrow.req.bits := out_buf - - val recv_cnt = Reg() { UFix(width = log2up(rbits)) } - val data_recv_cnt = Reg(resetVal = UFix(0, log2up(MEM_DATA_BITS))) + val recv_cnt = Reg(resetVal = UFix(0, log2up((rbits+w-1)/w))) + val data_recv_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) val resp_val = Reg(resetVal = Bool(false)) resp_val := Bool(false) @@ -80,10 +79,80 @@ class MemSerdes(w: Int) extends Component data_recv_cnt := data_recv_cnt + UFix(1) resp_val := Bool(true) } - in_buf := Cat(io.narrow.resp.bits, in_buf(rbits-1,w)) + in_buf := Cat(io.narrow.resp.bits, in_buf((rbits+w-1)/w*w-1,w)) } io.wide.resp.valid := resp_val io.wide.resp.bits.tag := in_buf(io.wide.resp.bits.tag.width-1,0) io.wide.resp.bits.data := in_buf >> UFix(io.wide.resp.bits.tag.width) } + +class MemDessert(w: Int) extends Component // test rig side +{ + val io = new Bundle { + val narrow = new ioMemSerialized(w).flip + val wide = new ioMem + } + val abits = io.wide.req_cmd.bits.toBits.getWidth + val dbits = io.wide.req_data.bits.toBits.getWidth + val rbits = io.wide.resp.bits.getWidth + + require(dbits >= abits && rbits >= dbits) + val recv_cnt = Reg(resetVal = UFix(0, log2up(rbits))) + val data_recv_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) + val adone = io.narrow.req.valid && recv_cnt === UFix((abits-1)/w) + val ddone = io.narrow.req.valid && recv_cnt === UFix((dbits-1)/w) + val rdone = io.narrow.resp.valid && recv_cnt === UFix((rbits-1)/w) + + val s_cmd_recv :: s_cmd :: s_data_recv :: s_data :: s_reply :: Nil = Enum(5) { UFix() } + val state = Reg(resetVal = s_cmd_recv) + + val in_buf = Reg() { Bits() } + when (io.narrow.req.valid && io.narrow.req.ready || io.narrow.resp.valid) { + recv_cnt := recv_cnt + UFix(1) + in_buf := Cat(io.narrow.req.bits, in_buf((rbits+w-1)/w*w-1,w)) + } + io.narrow.req.ready := state === s_cmd_recv || state === s_data_recv + + when (state === s_cmd_recv && adone) { + state := s_cmd + recv_cnt := UFix(0) + } + when (state === s_cmd && io.wide.req_cmd.ready) { + state := Mux(io.wide.req_cmd.bits.rw, s_data_recv, s_reply) + } + when (state === s_data_recv && ddone) { + state := s_data + recv_cnt := UFix(0) + } + when (state === s_data && io.wide.req_data.ready) { + state := s_data_recv + when (data_recv_cnt === UFix(REFILL_CYCLES-1)) { + state := s_cmd_recv + } + data_recv_cnt := data_recv_cnt + UFix(1) + } + when (rdone) { // state === s_reply + when (data_recv_cnt === UFix(REFILL_CYCLES-1)) { + state := s_cmd_recv + } + recv_cnt := UFix(0) + data_recv_cnt := data_recv_cnt + UFix(1) + } + + val req_cmd = in_buf >> UFix(((rbits+w-1)/w - (abits+w-1)/w)*w) + io.wide.req_cmd.valid := state === s_cmd + io.wide.req_cmd.bits.tag := req_cmd + io.wide.req_cmd.bits.addr := req_cmd.toUFix >> UFix(io.wide.req_cmd.bits.tag.width) + io.wide.req_cmd.bits.rw := req_cmd(io.wide.req_cmd.bits.tag.width + io.wide.req_cmd.bits.addr.width) + + io.wide.req_data.valid := state === s_data + io.wide.req_data.bits.data := in_buf >> UFix(((rbits+w-1)/w - (dbits+w-1)/w)*w) + + val dataq = (new queue(REFILL_CYCLES)) { new MemResp } + dataq.io.enq <> io.wide.resp + dataq.io.deq.ready := recv_cnt === UFix((rbits-1)/w) + + io.narrow.resp.valid := dataq.io.deq.valid + io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UFix(w)) +} From 25fe46dc18554e2a3bf78d977a943b64258535c9 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 26 Mar 2012 14:18:57 -0700 Subject: [PATCH 071/374] remove bug from dessert --- uncore/memserdes.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/memserdes.scala b/uncore/memserdes.scala index f68d36e8..c1d1fccb 100644 --- a/uncore/memserdes.scala +++ b/uncore/memserdes.scala @@ -26,7 +26,7 @@ class MemSerdes(w: Int) extends Component val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(5) { UFix() } val state = Reg(resetVal = s_idle) - val send_cnt = Reg(resetVal = UFix(0, log2up(max(abits, dbits)))) + val send_cnt = Reg(resetVal = UFix(0, log2up((max(abits, dbits)+w-1)/w))) val data_send_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) val adone = io.narrow.req.ready && send_cnt === UFix((abits-1)/w) val ddone = io.narrow.req.ready && send_cnt === UFix((dbits-1)/w) @@ -98,7 +98,7 @@ class MemDessert(w: Int) extends Component // test rig side val rbits = io.wide.resp.bits.getWidth require(dbits >= abits && rbits >= dbits) - val recv_cnt = Reg(resetVal = UFix(0, log2up(rbits))) + val recv_cnt = Reg(resetVal = UFix(0, log2up((rbits+w-1)/w))) val data_recv_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) val adone = io.narrow.req.valid && recv_cnt === UFix((abits-1)/w) val ddone = io.narrow.req.valid && recv_cnt === UFix((dbits-1)/w) From 257747a3a149f3f90d9645ab3a2cd3b5ca10b09e Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 26 Mar 2012 23:50:09 -0700 Subject: [PATCH 072/374] no dessert tonight :( --- uncore/memserdes.scala | 46 +++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/uncore/memserdes.scala b/uncore/memserdes.scala index c1d1fccb..27bbaa95 100644 --- a/uncore/memserdes.scala +++ b/uncore/memserdes.scala @@ -5,17 +5,17 @@ import Node._ import Constants._ import scala.math._ -class ioMemSerialized(w: Int) extends Bundle +class ioMemSerialized extends Bundle { - val req = (new ioDecoupled) { Bits(width = w) } - val resp = (new ioPipe) { Bits(width = w) }.flip + val req = (new ioDecoupled) { Bits(width = MEM_BACKUP_WIDTH) } + val resp = (new ioPipe) { Bits(width = MEM_BACKUP_WIDTH) }.flip } -class MemSerdes(w: Int) extends Component +class MemSerdes extends Component { val io = new Bundle { val wide = new ioMem().flip - val narrow = new ioMemSerialized(w) + val narrow = new ioMemSerialized } val abits = io.wide.req_cmd.bits.toBits.getWidth val dbits = io.wide.req_data.bits.toBits.getWidth @@ -26,14 +26,14 @@ class MemSerdes(w: Int) extends Component val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(5) { UFix() } val state = Reg(resetVal = s_idle) - val send_cnt = Reg(resetVal = UFix(0, log2up((max(abits, dbits)+w-1)/w))) + val send_cnt = Reg(resetVal = UFix(0, log2up((max(abits, dbits)+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) val data_send_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) - val adone = io.narrow.req.ready && send_cnt === UFix((abits-1)/w) - val ddone = io.narrow.req.ready && send_cnt === UFix((dbits-1)/w) + val adone = io.narrow.req.ready && send_cnt === UFix((abits-1)/MEM_BACKUP_WIDTH) + val ddone = io.narrow.req.ready && send_cnt === UFix((dbits-1)/MEM_BACKUP_WIDTH) when (io.narrow.req.valid && io.narrow.req.ready) { send_cnt := send_cnt + UFix(1) - out_buf := out_buf >> UFix(w) + out_buf := out_buf >> UFix(MEM_BACKUP_WIDTH) } when (io.wide.req_cmd.valid && io.wide.req_cmd.ready) { out_buf := io.wide.req_cmd.bits.toBits @@ -67,19 +67,19 @@ class MemSerdes(w: Int) extends Component send_cnt := UFix(0) } - val recv_cnt = Reg(resetVal = UFix(0, log2up((rbits+w-1)/w))) + val recv_cnt = Reg(resetVal = UFix(0, log2up((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) val data_recv_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) val resp_val = Reg(resetVal = Bool(false)) resp_val := Bool(false) when (io.narrow.resp.valid) { recv_cnt := recv_cnt + UFix(1) - when (recv_cnt === UFix((rbits-1)/w)) { + when (recv_cnt === UFix((rbits-1)/MEM_BACKUP_WIDTH)) { recv_cnt := UFix(0) data_recv_cnt := data_recv_cnt + UFix(1) resp_val := Bool(true) } - in_buf := Cat(io.narrow.resp.bits, in_buf((rbits+w-1)/w*w-1,w)) + in_buf := Cat(io.narrow.resp.bits, in_buf((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH*MEM_BACKUP_WIDTH-1,MEM_BACKUP_WIDTH)) } io.wide.resp.valid := resp_val @@ -87,10 +87,10 @@ class MemSerdes(w: Int) extends Component io.wide.resp.bits.data := in_buf >> UFix(io.wide.resp.bits.tag.width) } -class MemDessert(w: Int) extends Component // test rig side +class MemDessert extends Component // test rig side { val io = new Bundle { - val narrow = new ioMemSerialized(w).flip + val narrow = new ioMemSerialized().flip val wide = new ioMem } val abits = io.wide.req_cmd.bits.toBits.getWidth @@ -98,11 +98,11 @@ class MemDessert(w: Int) extends Component // test rig side val rbits = io.wide.resp.bits.getWidth require(dbits >= abits && rbits >= dbits) - val recv_cnt = Reg(resetVal = UFix(0, log2up((rbits+w-1)/w))) + val recv_cnt = Reg(resetVal = UFix(0, log2up((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) val data_recv_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) - val adone = io.narrow.req.valid && recv_cnt === UFix((abits-1)/w) - val ddone = io.narrow.req.valid && recv_cnt === UFix((dbits-1)/w) - val rdone = io.narrow.resp.valid && recv_cnt === UFix((rbits-1)/w) + val adone = io.narrow.req.valid && recv_cnt === UFix((abits-1)/MEM_BACKUP_WIDTH) + val ddone = io.narrow.req.valid && recv_cnt === UFix((dbits-1)/MEM_BACKUP_WIDTH) + val rdone = io.narrow.resp.valid && recv_cnt === UFix((rbits-1)/MEM_BACKUP_WIDTH) val s_cmd_recv :: s_cmd :: s_data_recv :: s_data :: s_reply :: Nil = Enum(5) { UFix() } val state = Reg(resetVal = s_cmd_recv) @@ -110,7 +110,7 @@ class MemDessert(w: Int) extends Component // test rig side val in_buf = Reg() { Bits() } when (io.narrow.req.valid && io.narrow.req.ready || io.narrow.resp.valid) { recv_cnt := recv_cnt + UFix(1) - in_buf := Cat(io.narrow.req.bits, in_buf((rbits+w-1)/w*w-1,w)) + in_buf := Cat(io.narrow.req.bits, in_buf((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH*MEM_BACKUP_WIDTH-1,MEM_BACKUP_WIDTH)) } io.narrow.req.ready := state === s_cmd_recv || state === s_data_recv @@ -140,19 +140,19 @@ class MemDessert(w: Int) extends Component // test rig side data_recv_cnt := data_recv_cnt + UFix(1) } - val req_cmd = in_buf >> UFix(((rbits+w-1)/w - (abits+w-1)/w)*w) + val req_cmd = in_buf >> UFix(((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH - (abits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH)*MEM_BACKUP_WIDTH) io.wide.req_cmd.valid := state === s_cmd io.wide.req_cmd.bits.tag := req_cmd io.wide.req_cmd.bits.addr := req_cmd.toUFix >> UFix(io.wide.req_cmd.bits.tag.width) io.wide.req_cmd.bits.rw := req_cmd(io.wide.req_cmd.bits.tag.width + io.wide.req_cmd.bits.addr.width) io.wide.req_data.valid := state === s_data - io.wide.req_data.bits.data := in_buf >> UFix(((rbits+w-1)/w - (dbits+w-1)/w)*w) + io.wide.req_data.bits.data := in_buf >> UFix(((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH - (dbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH)*MEM_BACKUP_WIDTH) val dataq = (new queue(REFILL_CYCLES)) { new MemResp } dataq.io.enq <> io.wide.resp - dataq.io.deq.ready := recv_cnt === UFix((rbits-1)/w) + dataq.io.deq.ready := recv_cnt === UFix((rbits-1)/MEM_BACKUP_WIDTH) io.narrow.resp.valid := dataq.io.deq.valid - io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UFix(w)) + io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UFix(MEM_BACKUP_WIDTH)) } From 73d8d42515680d903b18e2a05ecd593dc55d73e9 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 9 Apr 2012 21:40:35 -0700 Subject: [PATCH 073/374] fix coherence bug with multiple probe replies --- uncore/coherence.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 9cd8fc7c..678f83ab 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -598,8 +598,8 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS p_rep_data_dep_list(j).io.enq.valid := do_pop p_rep_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_p_reps) p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) - when (p_rep.valid) { - p_data_valid_arr(idx) := probeReplyHasData(p_rep.bits) + when (p_rep.valid && probeReplyHasData(p_rep.bits)) { + p_data_valid_arr(idx) := Bool(true) p_data_tile_id_arr(idx) := UFix(j) } p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) From 27e3346c14778bac96b87ec9ecea7f1118dd781f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 3 Apr 2012 12:03:05 -0700 Subject: [PATCH 074/374] Refactored coherence better from uncore hub, better coherence function names --- uncore/coherence.scala | 632 ++++++----------------------------------- uncore/uncore.scala | 503 ++++++++++++++++++++++++++++++++ 2 files changed, 591 insertions(+), 544 deletions(-) create mode 100644 uncore/uncore.scala diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 678f83ab..ee51e469 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -3,42 +3,6 @@ package rocket import Chisel._ import Constants._ -class MemData extends Bundle { - val data = Bits(width = MEM_DATA_BITS) -} - -class MemReqCmd() extends Bundle -{ - val rw = Bool() - val addr = UFix(width = PADDR_BITS - OFFSET_BITS) - val tag = Bits(width = MEM_TAG_BITS) -} - -class MemResp () extends MemData -{ - val tag = Bits(width = MEM_TAG_BITS) -} - -class ioMem() extends Bundle -{ - val req_cmd = (new ioDecoupled) { new MemReqCmd() } - val req_data = (new ioDecoupled) { new MemData() } - val resp = (new ioPipe) { new MemResp() }.flip -} - -class TrackerProbeData extends Bundle { - val tile_id = Bits(width = TILE_ID_BITS) -} - -class TrackerAllocReq extends Bundle { - val xact_init = new TransactionInit() - val tile_id = Bits(width = TILE_ID_BITS) -} - -class TrackerDependency extends Bundle { - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) -} - class TransactionInit extends Bundle { val t_type = Bits(width = X_INIT_TYPE_BITS) val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) @@ -75,17 +39,6 @@ class TransactionFinish extends Bundle { val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } -class ioTileLink extends Bundle { - val xact_init = (new ioDecoupled) { new TransactionInit() } - val xact_init_data = (new ioDecoupled) { new TransactionInitData() } - val xact_abort = (new ioDecoupled) { new TransactionAbort() }.flip - val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip - val probe_rep = (new ioDecoupled) { new ProbeReply() } - val probe_rep_data = (new ioDecoupled) { new ProbeReplyData() } - val xact_rep = (new ioPipe) { new TransactionReply() }.flip - val xact_finish = (new ioDecoupled) { new TransactionFinish() } -} - object cpuCmdToRW { def apply(cmd: Bits): (Bool, Bool) = { val store = (cmd === M_XWR) @@ -101,6 +54,13 @@ trait CoherencePolicy { } trait ThreeStateIncoherence extends CoherencePolicy { val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } + val X_INIT_READ_SHARED = UFix(0, 2) + val X_INIT_READ_EXCLUSIVE = UFix(1, 2) + val X_INIT_WRITE_UNCACHED = UFix(3, 2) + val X_REP_READ_SHARED = UFix(0, X_REP_TYPE_BITS) + val X_REP_READ_EXCLUSIVE = UFix(1, X_REP_TYPE_BITS) + val X_REP_WRITE_UNCACHED = UFix(3, X_REP_TYPE_BITS) + val P_REP_INVALIDATE_ACK = UFix(3, P_REP_TYPE_BITS) def isHit ( cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) @@ -116,25 +76,25 @@ trait ThreeStateIncoherence extends CoherencePolicy { } def newStateOnWriteback() = tileInvalid - def newStateOnFlush() = tileInvalid + def newStateOnCacheControl(cmd: Bits) = tileInvalid def newState(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) Mux(write, tileDirty, Mux(read, Mux(state === tileDirty, tileDirty, tileClean), state)) } def newStateOnHit(cmd: Bits, state: UFix): UFix = newState(cmd, state) - def newTransactionOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) Mux(write || cmd === M_PFW, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) } - def newTransactionOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { val (read, write) = cpuCmdToRW(cmd) Mux(write, X_INIT_READ_EXCLUSIVE, outstanding.t_type) } - def needsSecondaryXact(cmd: Bits, outstanding: TransactionInit): Bool = Bool(false) - def newStateOnTransactionRep(incoming: TransactionReply, outstanding: TransactionInit): UFix = { + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = Bool(false) + def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { Mux(outstanding.t_type === X_INIT_READ_EXCLUSIVE, tileDirty, tileClean) } - def newStateOnProbeReq(incoming: ProbeRequest, state: UFix): Bits = state + def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = state def newProbeReply (incoming: ProbeRequest, has_data: Bool): ProbeReply = { val reply = Wire() { new ProbeReply() } reply.p_type := P_REP_INVALIDATE_ACK @@ -151,43 +111,66 @@ trait FourStateCoherence extends CoherencePolicy { val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(3){ UFix() } val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } + val X_INIT_READ_SHARED = UFix(0, X_INIT_TYPE_BITS) + val X_INIT_READ_EXCLUSIVE = UFix(1, X_INIT_TYPE_BITS) + val X_INIT_READ_UNCACHED = UFix(2, X_INIT_TYPE_BITS) + val X_INIT_WRITE_UNCACHED = UFix(3, X_INIT_TYPE_BITS) + + val X_REP_READ_SHARED = UFix(0, X_REP_TYPE_BITS) + val X_REP_READ_EXCLUSIVE = UFix(1, X_REP_TYPE_BITS) + val X_REP_READ_UNCACHED = UFix(2, X_REP_TYPE_BITS) + val X_REP_WRITE_UNCACHED = UFix(3, X_REP_TYPE_BITS) + val X_REP_READ_EXCLUSIVE_ACK = UFix(4, X_REP_TYPE_BITS) + + val P_REQ_INVALIDATE = UFix(0, P_REQ_TYPE_BITS) + val P_REQ_DOWNGRADE = UFix(1, P_REQ_TYPE_BITS) + val P_REQ_COPY = UFix(2, P_REQ_TYPE_BITS) + + val P_REP_INVALIDATE_DATA = UFix(0, P_REP_TYPE_BITS) + val P_REP_DOWNGRADE_DATA = UFix(1, P_REP_TYPE_BITS) + val P_REP_COPY_DATA = UFix(2, P_REP_TYPE_BITS) + val P_REP_INVALIDATE_ACK = UFix(3, P_REP_TYPE_BITS) + val P_REP_DOWNGRADE_ACK = UFix(4, P_REP_TYPE_BITS) + val P_REP_COPY_ACK = UFix(5, P_REP_TYPE_BITS) + + def isHit ( cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) Mux(write, (state === tileExclusiveClean || state === tileExclusiveDirty), (state === tileShared || state === tileExclusiveClean || state === tileExclusiveDirty)) } - - //TODO: do we need isPresent() for determining that a line needs to be - //upgraded but that no replacement is needed? - def isValid (state: UFix): Bool = { state != tileInvalid } - def needsWriteback (state: UFix): Bool = { - state === tileExclusiveDirty - } - - def newStateOnWriteback() = tileInvalid - def newStateOnFlush() = tileInvalid - def newStateOnHit(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, tileExclusiveDirty, state) - } - def newTransactionOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) - } - def newTransactionOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, X_INIT_READ_EXCLUSIVE, outstanding.t_type) - } - def needsSecondaryXact(cmd: Bits, outstanding: TransactionInit): Bool = { + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { val (read, write) = cpuCmdToRW(cmd) (read && (outstanding.t_type === X_INIT_READ_UNCACHED || outstanding.t_type === X_INIT_WRITE_UNCACHED)) || (write && (outstanding.t_type != X_INIT_READ_EXCLUSIVE)) } - def newStateOnTransactionRep(incoming: TransactionReply, outstanding: TransactionInit): UFix = { + def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { + MuxLookup(cmd, (state === tileExclusiveDirty), Array( + M_INV -> (state === tileExclusiveDirty), + M_CLN -> (state === tileExclusiveDirty) + )) + } + def needsWriteback (state: UFix): Bool = { + needsTransactionOnCacheControl(M_INV, state) + } + + def newStateOnHit(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, tileExclusiveDirty, state) + } + def newStateOnCacheControl(cmd: Bits) = { + MuxLookup(cmd, tileInvalid, Array( + M_INV -> tileInvalid, + M_CLN -> tileExclusiveClean + )) + } + def newStateOnWriteback() = newStateOnCacheControl(M_INV) + def newStateOnFlush() = newStateOnCacheControl(M_INV) + def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { MuxLookup(incoming.t_type, tileInvalid, Array( X_REP_READ_SHARED -> tileShared, X_REP_READ_EXCLUSIVE -> Mux(outstanding.t_type === X_INIT_READ_EXCLUSIVE, tileExclusiveDirty, tileExclusiveClean), @@ -196,8 +179,7 @@ trait FourStateCoherence extends CoherencePolicy { X_REP_WRITE_UNCACHED -> tileInvalid )) } - - def newStateOnProbeReq(incoming: ProbeRequest, state: UFix): Bits = { + def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeDowngrade -> tileShared, @@ -205,6 +187,17 @@ trait FourStateCoherence extends CoherencePolicy { )) } + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write || cmd === M_PFW, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) + } + def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, X_INIT_READ_EXCLUSIVE, outstanding.t_type) + } + def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = X_INIT_WRITE_UNCACHED + def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) + def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { val reply = Wire() { new ProbeReply() } val with_data = MuxLookup(incoming.p_type, P_REP_INVALIDATE_DATA, Array( @@ -221,55 +214,31 @@ trait FourStateCoherence extends CoherencePolicy { reply.global_xact_id := incoming.global_xact_id reply } - def probeReplyHasData (reply: ProbeReply): Bool = { + + def hasData (reply: ProbeReply): Bool = { (reply.p_type === P_REP_INVALIDATE_DATA || reply.p_type === P_REP_DOWNGRADE_DATA || reply.p_type === P_REP_COPY_DATA) } - def transactionInitHasData (init: TransactionInit): Bool = { + def hasData (init: TransactionInit): Bool = { (init.t_type === X_INIT_WRITE_UNCACHED) } - def transactionReplyHasData (reply: TransactionReply): Bool = { + def hasData (reply: TransactionReply): Bool = { (reply.t_type != X_REP_WRITE_UNCACHED && reply.t_type != X_REP_READ_EXCLUSIVE_ACK) } -} -class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherence { - val io = new Bundle { - val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip - val p_data = (new ioPipe) { new TrackerProbeData }.flip - val can_alloc = Bool(INPUT) - val xact_finish = Bool(INPUT) - val p_rep_cnt_dec = Bits(ntiles, INPUT) - val p_req_cnt_inc = Bits(ntiles, INPUT) - val p_rep_data = (new ioPipe) { new ProbeReplyData }.flip - val x_init_data = (new ioPipe) { new TransactionInitData }.flip - val sent_x_rep_ack = Bool(INPUT) - val p_rep_data_dep = (new ioPipe) { new TrackerDependency }.flip - val x_init_data_dep = (new ioPipe) { new TrackerDependency }.flip + def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - val mem_req_cmd = (new ioDecoupled) { new MemReqCmd } - val mem_req_data = (new ioDecoupled) { new MemData } - val mem_req_lock = Bool(OUTPUT) - val probe_req = (new ioDecoupled) { new ProbeRequest } - val busy = Bool(OUTPUT) - val addr = Bits(PADDR_BITS - OFFSET_BITS, OUTPUT) - val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) - val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) - val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) - val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) - val t_type = Bits(X_INIT_TYPE_BITS, OUTPUT) - val push_p_req = Bits(ntiles, OUTPUT) - val pop_p_rep = Bits(ntiles, OUTPUT) - val pop_p_rep_data = Bits(ntiles, OUTPUT) - val pop_p_rep_dep = Bits(ntiles, OUTPUT) - val pop_x_init = Bits(ntiles, OUTPUT) - val pop_x_init_data = Bits(ntiles, OUTPUT) - val pop_x_init_dep = Bits(ntiles, OUTPUT) - val send_x_rep_ack = Bool(OUTPUT) + def getTransactionReplyType(t_type: UFix, count: UFix): Bits = { + MuxLookup(t_type, X_REP_READ_UNCACHED, Array( + X_INIT_READ_SHARED -> Mux(count > UFix(0), X_REP_READ_SHARED, X_REP_READ_EXCLUSIVE), + X_INIT_READ_EXCLUSIVE -> X_REP_READ_EXCLUSIVE, + X_INIT_READ_UNCACHED -> X_REP_READ_UNCACHED, + X_INIT_WRITE_UNCACHED -> X_REP_WRITE_UNCACHED + )) } - def sendProbeReqType(t_type: UFix, global_state: UFix): UFix = { + def getProbeRequestType(t_type: UFix, global_state: UFix): UFix = { MuxLookup(t_type, P_REQ_COPY, Array( X_INIT_READ_SHARED -> P_REQ_DOWNGRADE, X_INIT_READ_EXCLUSIVE -> P_REQ_INVALIDATE, @@ -281,435 +250,10 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc def needsMemRead(t_type: UFix, global_state: UFix): Bool = { (t_type != X_INIT_WRITE_UNCACHED) } - - def needsAckRep(t_type: UFix, global_state: UFix): Bool = { + def needsMemWrite(t_type: UFix, global_state: UFix): Bool = { (t_type === X_INIT_WRITE_UNCACHED) } - - def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioPipe[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { - req_cmd.valid := !cmd_sent && at_front_of_dep_queue - req_cmd.bits.rw := Bool(true) - req_data.valid := data.valid && at_front_of_dep_queue - req_data.bits := data.bits - lock := at_front_of_dep_queue - when(req_cmd.ready && req_cmd.valid) { - cmd_sent := Bool(true) - } - when(req_data.ready && at_front_of_dep_queue) { - pop_data := UFix(1) << tile_id - when (data.valid) { - mem_cnt := mem_cnt_next - when(mem_cnt_next === UFix(0)) { - pop_dep := UFix(1) << tile_id - trigger := Bool(false) - } - } - } - } - - def doMemReqRead(req_cmd: ioDecoupled[MemReqCmd], trigger: Bool) { - req_cmd.valid := Bool(true) - req_cmd.bits.rw := Bool(false) - when(req_cmd.ready) { - trigger := Bool(false) - } - } - - val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } - val state = Reg(resetVal = s_idle) - val addr_ = Reg{ UFix() } - val t_type_ = Reg{ Bits() } - val init_tile_id_ = Reg{ Bits() } - val tile_xact_id_ = Reg{ Bits() } - val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2up(ntiles))) - val p_req_flags = Reg(resetVal = Bits(0, width = ntiles)) - val p_rep_tile_id_ = Reg{ Bits() } - val x_needs_read = Reg(resetVal = Bool(false)) - val x_init_data_needs_write = Reg(resetVal = Bool(false)) - val p_rep_data_needs_write = Reg(resetVal = Bool(false)) - val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val mem_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) - val mem_cnt_next = mem_cnt + UFix(1) - val mem_cnt_max = ~UFix(0, width = log2up(REFILL_CYCLES)) - - io.busy := state != s_idle - io.addr := addr_ - io.init_tile_id := init_tile_id_ - io.p_rep_tile_id := p_rep_tile_id_ - io.tile_xact_id := tile_xact_id_ - io.sharer_count := UFix(ntiles) // TODO: Broadcast only - io.t_type := t_type_ - - io.mem_req_cmd.valid := Bool(false) - io.mem_req_cmd.bits.rw := Bool(false) - io.mem_req_cmd.bits.addr := addr_ - io.mem_req_cmd.bits.tag := UFix(id) - io.mem_req_data.valid := Bool(false) - io.mem_req_data.bits.data := UFix(0) - io.mem_req_lock := Bool(false) - io.probe_req.valid := Bool(false) - io.probe_req.bits.p_type := sendProbeReqType(t_type_, UFix(0)) - io.probe_req.bits.global_xact_id := UFix(id) - io.probe_req.bits.address := addr_ - io.push_p_req := Bits(0, width = ntiles) - io.pop_p_rep := Bits(0, width = ntiles) - io.pop_p_rep_data := Bits(0, width = ntiles) - io.pop_p_rep_dep := Bits(0, width = ntiles) - io.pop_x_init := Bits(0, width = ntiles) - io.pop_x_init_data := Bits(0, width = ntiles) - io.pop_x_init_dep := Bits(0, width = ntiles) - io.send_x_rep_ack := Bool(false) - - switch (state) { - is(s_idle) { - when( io.alloc_req.valid && io.can_alloc ) { - addr_ := io.alloc_req.bits.xact_init.address - t_type_ := io.alloc_req.bits.xact_init.t_type - init_tile_id_ := io.alloc_req.bits.tile_id - tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id - x_init_data_needs_write := transactionInitHasData(io.alloc_req.bits.xact_init) - x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) - if(ntiles > 1) p_rep_count := UFix(ntiles-1) - val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only - p_req_flags := p_req_initial_flags - mem_cnt := UFix(0) - p_w_mem_cmd_sent := Bool(false) - x_w_mem_cmd_sent := Bool(false) - io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id - state := Mux(p_req_initial_flags.orR, s_probe, s_mem) - } - } - is(s_probe) { - when(p_req_flags.orR) { - io.push_p_req := p_req_flags - io.probe_req.valid := Bool(true) - } - when(io.p_req_cnt_inc.orR) { - p_req_flags := p_req_flags & ~io.p_req_cnt_inc // unflag sent reqs - } - when(io.p_rep_cnt_dec.orR) { - val dec = PopCount(io.p_rep_cnt_dec) - io.pop_p_rep := io.p_rep_cnt_dec - if(ntiles > 1) p_rep_count := p_rep_count - dec - when(p_rep_count === dec) { - state := s_mem - } - } - when(io.p_data.valid) { - p_rep_data_needs_write := Bool(true) - p_rep_tile_id_ := io.p_data.bits.tile_id - } - } - is(s_mem) { - when (p_rep_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, - io.mem_req_data, - io.mem_req_lock, - io.p_rep_data, - p_rep_data_needs_write, - p_w_mem_cmd_sent, - io.pop_p_rep_data, - io.pop_p_rep_dep, - io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id)), - p_rep_tile_id_) - } . elsewhen(x_init_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, - io.mem_req_data, - io.mem_req_lock, - io.x_init_data, - x_init_data_needs_write, - x_w_mem_cmd_sent, - io.pop_x_init_data, - io.pop_x_init_dep, - io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id)), - init_tile_id_) - } . elsewhen (x_needs_read) { - doMemReqRead(io.mem_req_cmd, x_needs_read) - } . otherwise { - state := Mux(needsAckRep(t_type_, UFix(0)), s_ack, s_busy) - } - } - is(s_ack) { - io.send_x_rep_ack := Bool(true) - when(io.sent_x_rep_ack) { state := s_busy } - } - is(s_busy) { // Nothing left to do but wait for transaction to complete - when (io.xact_finish) { - state := s_idle - } - } + def needsAckReply(t_type: UFix, global_state: UFix): Bool = { + (t_type === X_INIT_WRITE_UNCACHED) } } - -abstract class CoherenceHub(ntiles: Int) extends Component with CoherencePolicy { - val io = new Bundle { - val tiles = Vec(ntiles) { new ioTileLink() }.flip - val mem = new ioMem - } -} - -class CoherenceHubNull extends CoherenceHub(1) { - - val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.t_type === X_INIT_WRITE_UNCACHED - x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp - io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) - io.mem.req_cmd.bits.rw := is_write - io.mem.req_cmd.bits.tag := x_init.bits.tile_xact_id - io.mem.req_cmd.bits.addr := x_init.bits.address - io.mem.req_data <> io.tiles(0).xact_init_data - - val x_rep = io.tiles(0).xact_rep - x_rep.bits.t_type := Mux(io.mem.resp.valid, X_REP_READ_EXCLUSIVE, X_REP_WRITE_UNCACHED) - x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) - x_rep.bits.global_xact_id := UFix(0) // don't care - x_rep.bits.data := io.mem.resp.bits.data - x_rep.bits.require_ack := Bool(true) - x_rep.valid := io.mem.resp.valid || x_init.valid && is_write && io.mem.req_cmd.ready - - io.tiles(0).xact_abort.valid := Bool(false) - io.tiles(0).xact_finish.ready := Bool(true) - io.tiles(0).probe_req.valid := Bool(false) - io.tiles(0).probe_rep.ready := Bool(true) - io.tiles(0).probe_rep_data.ready := Bool(true) -} - - -class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourStateCoherence{ - - def coherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - - def getTransactionReplyType(t_type: UFix, count: UFix): Bits = { - MuxLookup(t_type, X_REP_READ_UNCACHED, Array( - X_INIT_READ_SHARED -> Mux(count > UFix(0), X_REP_READ_SHARED, X_REP_READ_EXCLUSIVE), - X_INIT_READ_EXCLUSIVE -> X_REP_READ_EXCLUSIVE, - X_INIT_READ_UNCACHED -> X_REP_READ_UNCACHED, - X_INIT_WRITE_UNCACHED -> X_REP_WRITE_UNCACHED - )) - } - - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _)) - - val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } - val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } - val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - - val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } - val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } - val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } - val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bits(width = TILE_ID_BITS)} } - val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } - - for( i <- 0 until NGLOBAL_XACTS) { - val t = trackerList(i).io - busy_arr(i) := t.busy - addr_arr(i) := t.addr - init_tile_id_arr(i) := t.init_tile_id - tile_xact_id_arr(i) := t.tile_xact_id - t_type_arr(i) := t.t_type - sh_count_arr(i) := t.sharer_count - send_x_rep_ack_arr(i) := t.send_x_rep_ack - t.xact_finish := do_free_arr(i) - t.p_data.bits.tile_id := p_data_tile_id_arr(i) - t.p_data.valid := p_data_valid_arr(i) - t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits - t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits - t.sent_x_rep_ack := sent_x_rep_ack_arr(i) - do_free_arr(i) := Bool(false) - sent_x_rep_ack_arr(i) := Bool(false) - p_data_tile_id_arr(i) := Bits(0, width = TILE_ID_BITS) - p_data_valid_arr(i) := Bool(false) - for( j <- 0 until ntiles) { - p_rep_cnt_dec_arr(i)(j) := Bool(false) - p_req_cnt_inc_arr(i)(j) := Bool(false) - } - } - - val p_rep_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY - val x_init_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY - - // Free finished transactions - for( j <- 0 until ntiles ) { - val finish = io.tiles(j).xact_finish - when (finish.valid) { - do_free_arr(finish.bits.global_xact_id) := Bool(true) - } - finish.ready := Bool(true) - } - - // Reply to initial requestor - // Forward memory responses from mem to tile or arbitrate to ack - val mem_idx = io.mem.resp.bits.tag - val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) - for( j <- 0 until ntiles ) { - val rep = io.tiles(j).xact_rep - rep.bits.t_type := UFix(0) - rep.bits.tile_xact_id := UFix(0) - rep.bits.global_xact_id := UFix(0) - rep.bits.data := io.mem.resp.bits.data - rep.bits.require_ack := Bool(true) - rep.valid := Bool(false) - when(io.mem.resp.valid && (UFix(j) === init_tile_id_arr(mem_idx))) { - rep.bits.t_type := getTransactionReplyType(t_type_arr(mem_idx), sh_count_arr(mem_idx)) - rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) - rep.bits.global_xact_id := mem_idx - rep.valid := Bool(true) - } . otherwise { - rep.bits.t_type := getTransactionReplyType(t_type_arr(ack_idx), sh_count_arr(ack_idx)) - rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) - rep.bits.global_xact_id := ack_idx - when (UFix(j) === init_tile_id_arr(ack_idx)) { - rep.valid := send_x_rep_ack_arr.toBits.orR - sent_x_rep_ack_arr(ack_idx) := Bool(true) - } - } - } - // If there were a ready signal due to e.g. intervening network use: - //io.mem.resp.ready := io.tiles(init_tile_id_arr.read(mem_idx)).xact_rep.ready - - // Create an arbiter for the one memory port - // We have to arbitrate between the different trackers' memory requests - // and once we have picked a request, get the right write data - val mem_req_cmd_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemReqCmd() } - val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemData() } - for( i <- 0 until NGLOBAL_XACTS ) { - mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd - mem_req_cmd_arb.io.lock(i) <> trackerList(i).io.mem_req_lock - mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data - mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock - } - io.mem.req_cmd <> mem_req_cmd_arb.io.out - io.mem.req_data <> mem_req_data_arb.io.out - - // Handle probe replies, which may or may not have data - for( j <- 0 until ntiles ) { - val p_rep = io.tiles(j).probe_rep - val p_rep_data = io.tiles(j).probe_rep_data - val idx = p_rep.bits.global_xact_id - val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) - val do_pop = foldR(pop_p_reps)(_ || _) - p_rep.ready := Bool(true) - p_rep_data_dep_list(j).io.enq.valid := do_pop - p_rep_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_p_reps) - p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) - when (p_rep.valid && probeReplyHasData(p_rep.bits)) { - p_data_valid_arr(idx) := Bool(true) - p_data_tile_id_arr(idx) := UFix(j) - } - p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) - } - for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid - trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits - - trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) - trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) - - for( j <- 0 until ntiles) { - val p_rep = io.tiles(j).probe_rep - p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) - } - } - - // Nack conflicting transaction init attempts - val s_idle :: s_abort_drain :: s_abort_send :: s_abort_complete :: Nil = Enum(4){ UFix() } - val abort_state_arr = Vec(ntiles) { Reg(resetVal = s_idle) } - val want_to_abort_arr = Vec(ntiles) { Wire() { Bool()} } - for( j <- 0 until ntiles ) { - val x_init = io.tiles(j).xact_init - val x_init_data = io.tiles(j).xact_init_data - val x_abort = io.tiles(j).xact_abort - val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) - val conflicts = Vec(NGLOBAL_XACTS) { Wire() { Bool() } } - for( i <- 0 until NGLOBAL_XACTS) { - val t = trackerList(i).io - conflicts(i) := t.busy && x_init.valid && coherenceConflict(t.addr, x_init.bits.address) - } - x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id - want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && transactionInitHasData(x_init.bits))) - - x_abort.valid := Bool(false) - switch(abort_state_arr(j)) { - is(s_idle) { - when(want_to_abort_arr(j)) { - when(transactionInitHasData(x_init.bits)) { - abort_state_arr(j) := s_abort_drain - } . otherwise { - abort_state_arr(j) := s_abort_send - } - } - } - is(s_abort_drain) { // raises x_init_data.ready below - when(x_init_data.valid) { - abort_cnt := abort_cnt + UFix(1) - when(abort_cnt === ~UFix(0, width = log2up(REFILL_CYCLES))) { - abort_state_arr(j) := s_abort_send - } - } - } - is(s_abort_send) { // nothing is dequeued for now - x_abort.valid := Bool(true) - when(x_abort.ready) { - abort_state_arr(j) := s_abort_complete - } - } - is(s_abort_complete) { // raises x_init.ready below - abort_state_arr(j) := s_idle - } - } - } - - // Handle transaction initiation requests - // Only one allocation per cycle - // Init requests may or may not have data - val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } - val init_arb = (new Arbiter(ntiles)) { new TrackerAllocReq() } - for( i <- 0 until NGLOBAL_XACTS ) { - alloc_arb.io.in(i).valid := !trackerList(i).io.busy - trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready - trackerList(i).io.alloc_req.bits := init_arb.io.out.bits - trackerList(i).io.alloc_req.valid := init_arb.io.out.valid - - trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits - trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid - trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) - trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) - } - for( j <- 0 until ntiles ) { - val x_init = io.tiles(j).xact_init - val x_init_data = io.tiles(j).xact_init_data - val x_init_data_dep = x_init_data_dep_list(j).io.deq - init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid - init_arb.io.in(j).bits.xact_init := x_init.bits - init_arb.io.in(j).bits.tile_id := UFix(j) - val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) - val do_pop = foldR(pop_x_inits)(_||_) - x_init_data_dep_list(j).io.enq.valid := do_pop && transactionInitHasData(x_init.bits) && (abort_state_arr(j) === s_idle) - x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) - x_init.ready := (abort_state_arr(j) === s_abort_complete) || do_pop - x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) - x_init_data_dep.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) - } - - alloc_arb.io.out.ready := init_arb.io.out.valid - - // Handle probe request generation - // Must arbitrate for each request port - val p_req_arb_arr = List.fill(ntiles)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) - for( j <- 0 until ntiles ) { - for( i <- 0 until NGLOBAL_XACTS ) { - val t = trackerList(i).io - p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits - p_req_arb_arr(j).io.in(i).valid := t.probe_req.valid && t.push_p_req(j) - p_req_cnt_inc_arr(i)(j) := p_req_arb_arr(j).io.in(i).ready - } - p_req_arb_arr(j).io.out <> io.tiles(j).probe_req - } - -} diff --git a/uncore/uncore.scala b/uncore/uncore.scala new file mode 100644 index 00000000..fa6e2b1e --- /dev/null +++ b/uncore/uncore.scala @@ -0,0 +1,503 @@ +package rocket + +import Chisel._ +import Constants._ + +class MemData extends Bundle { + val data = Bits(width = MEM_DATA_BITS) +} + +class MemReqCmd() extends Bundle +{ + val rw = Bool() + val addr = UFix(width = PADDR_BITS - OFFSET_BITS) + val tag = Bits(width = MEM_TAG_BITS) +} + +class MemResp () extends MemData +{ + val tag = Bits(width = MEM_TAG_BITS) +} + +class ioMem() extends Bundle +{ + val req_cmd = (new ioDecoupled) { new MemReqCmd() } + val req_data = (new ioDecoupled) { new MemData() } + val resp = (new ioPipe) { new MemResp() }.flip +} + +class TrackerProbeData extends Bundle { + val tile_id = Bits(width = TILE_ID_BITS) +} + +class TrackerAllocReq extends Bundle { + val xact_init = new TransactionInit() + val tile_id = Bits(width = TILE_ID_BITS) +} + +class TrackerDependency extends Bundle { + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) +} + +class ioTileLink extends Bundle { + val xact_init = (new ioDecoupled) { new TransactionInit } + val xact_init_data = (new ioDecoupled) { new TransactionInitData } + val xact_abort = (new ioDecoupled) { new TransactionAbort }.flip + val probe_req = (new ioDecoupled) { new ProbeRequest }.flip + val probe_rep = (new ioDecoupled) { new ProbeReply } + val probe_rep_data = (new ioDecoupled) { new ProbeReplyData } + val xact_rep = (new ioPipe) { new TransactionReply }.flip + val xact_finish = (new ioDecoupled) { new TransactionFinish } +} + +class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherence { + val io = new Bundle { + val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip + val p_data = (new ioPipe) { new TrackerProbeData }.flip + val can_alloc = Bool(INPUT) + val xact_finish = Bool(INPUT) + val p_rep_cnt_dec = Bits(ntiles, INPUT) + val p_req_cnt_inc = Bits(ntiles, INPUT) + val p_rep_data = (new ioPipe) { new ProbeReplyData }.flip + val x_init_data = (new ioPipe) { new TransactionInitData }.flip + val sent_x_rep_ack = Bool(INPUT) + val p_rep_data_dep = (new ioPipe) { new TrackerDependency }.flip + val x_init_data_dep = (new ioPipe) { new TrackerDependency }.flip + + val mem_req_cmd = (new ioDecoupled) { new MemReqCmd } + val mem_req_data = (new ioDecoupled) { new MemData } + val mem_req_lock = Bool(OUTPUT) + val probe_req = (new ioDecoupled) { new ProbeRequest } + val busy = Bool(OUTPUT) + val addr = Bits(PADDR_BITS - OFFSET_BITS, OUTPUT) + val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) + val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) + val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) + val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) + val t_type = Bits(X_INIT_TYPE_BITS, OUTPUT) + val push_p_req = Bits(ntiles, OUTPUT) + val pop_p_rep = Bits(ntiles, OUTPUT) + val pop_p_rep_data = Bits(ntiles, OUTPUT) + val pop_p_rep_dep = Bits(ntiles, OUTPUT) + val pop_x_init = Bits(ntiles, OUTPUT) + val pop_x_init_data = Bits(ntiles, OUTPUT) + val pop_x_init_dep = Bits(ntiles, OUTPUT) + val send_x_rep_ack = Bool(OUTPUT) + } + + def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioPipe[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { + req_cmd.valid := !cmd_sent && at_front_of_dep_queue + req_cmd.bits.rw := Bool(true) + req_data.valid := data.valid && at_front_of_dep_queue + req_data.bits := data.bits + lock := at_front_of_dep_queue + when(req_cmd.ready && req_cmd.valid) { + cmd_sent := Bool(true) + } + when(req_data.ready && at_front_of_dep_queue) { + pop_data := UFix(1) << tile_id + when (data.valid) { + mem_cnt := mem_cnt_next + when(mem_cnt_next === UFix(0)) { + pop_dep := UFix(1) << tile_id + trigger := Bool(false) + } + } + } + } + + def doMemReqRead(req_cmd: ioDecoupled[MemReqCmd], trigger: Bool) { + req_cmd.valid := Bool(true) + req_cmd.bits.rw := Bool(false) + when(req_cmd.ready) { + trigger := Bool(false) + } + } + + val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } + val state = Reg(resetVal = s_idle) + val addr_ = Reg{ UFix() } + val t_type_ = Reg{ Bits() } + val init_tile_id_ = Reg{ Bits() } + val tile_xact_id_ = Reg{ Bits() } + val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2up(ntiles))) + val p_req_flags = Reg(resetVal = Bits(0, width = ntiles)) + val p_rep_tile_id_ = Reg{ Bits() } + val x_needs_read = Reg(resetVal = Bool(false)) + val x_init_data_needs_write = Reg(resetVal = Bool(false)) + val p_rep_data_needs_write = Reg(resetVal = Bool(false)) + val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val mem_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val mem_cnt_next = mem_cnt + UFix(1) + val mem_cnt_max = ~UFix(0, width = log2up(REFILL_CYCLES)) + + io.busy := state != s_idle + io.addr := addr_ + io.init_tile_id := init_tile_id_ + io.p_rep_tile_id := p_rep_tile_id_ + io.tile_xact_id := tile_xact_id_ + io.sharer_count := UFix(ntiles) // TODO: Broadcast only + io.t_type := t_type_ + + io.mem_req_cmd.valid := Bool(false) + io.mem_req_cmd.bits.rw := Bool(false) + io.mem_req_cmd.bits.addr := addr_ + io.mem_req_cmd.bits.tag := UFix(id) + io.mem_req_data.valid := Bool(false) + io.mem_req_data.bits.data := UFix(0) + io.mem_req_lock := Bool(false) + io.probe_req.valid := Bool(false) + io.probe_req.bits.p_type := getProbeRequestType(t_type_, UFix(0)) + io.probe_req.bits.global_xact_id := UFix(id) + io.probe_req.bits.address := addr_ + io.push_p_req := Bits(0, width = ntiles) + io.pop_p_rep := Bits(0, width = ntiles) + io.pop_p_rep_data := Bits(0, width = ntiles) + io.pop_p_rep_dep := Bits(0, width = ntiles) + io.pop_x_init := Bits(0, width = ntiles) + io.pop_x_init_data := Bits(0, width = ntiles) + io.pop_x_init_dep := Bits(0, width = ntiles) + io.send_x_rep_ack := Bool(false) + + switch (state) { + is(s_idle) { + when( io.alloc_req.valid && io.can_alloc ) { + addr_ := io.alloc_req.bits.xact_init.address + t_type_ := io.alloc_req.bits.xact_init.t_type + init_tile_id_ := io.alloc_req.bits.tile_id + tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id + x_init_data_needs_write := hasData(io.alloc_req.bits.xact_init) + x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) + if(ntiles > 1) p_rep_count := UFix(ntiles-1) + val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only + p_req_flags := p_req_initial_flags + mem_cnt := UFix(0) + p_w_mem_cmd_sent := Bool(false) + x_w_mem_cmd_sent := Bool(false) + io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id + state := Mux(p_req_initial_flags.orR, s_probe, s_mem) + } + } + is(s_probe) { + when(p_req_flags.orR) { + io.push_p_req := p_req_flags + io.probe_req.valid := Bool(true) + } + when(io.p_req_cnt_inc.orR) { + p_req_flags := p_req_flags & ~io.p_req_cnt_inc // unflag sent reqs + } + when(io.p_rep_cnt_dec.orR) { + val dec = PopCount(io.p_rep_cnt_dec) + io.pop_p_rep := io.p_rep_cnt_dec + if(ntiles > 1) p_rep_count := p_rep_count - dec + when(p_rep_count === dec) { + state := s_mem + } + } + when(io.p_data.valid) { + p_rep_data_needs_write := Bool(true) + p_rep_tile_id_ := io.p_data.bits.tile_id + } + } + is(s_mem) { + when (p_rep_data_needs_write) { + doMemReqWrite(io.mem_req_cmd, + io.mem_req_data, + io.mem_req_lock, + io.p_rep_data, + p_rep_data_needs_write, + p_w_mem_cmd_sent, + io.pop_p_rep_data, + io.pop_p_rep_dep, + io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id)), + p_rep_tile_id_) + } . elsewhen(x_init_data_needs_write) { + doMemReqWrite(io.mem_req_cmd, + io.mem_req_data, + io.mem_req_lock, + io.x_init_data, + x_init_data_needs_write, + x_w_mem_cmd_sent, + io.pop_x_init_data, + io.pop_x_init_dep, + io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id)), + init_tile_id_) + } . elsewhen (x_needs_read) { + doMemReqRead(io.mem_req_cmd, x_needs_read) + } . otherwise { + state := Mux(needsAckReply(t_type_, UFix(0)), s_ack, s_busy) + } + } + is(s_ack) { + io.send_x_rep_ack := Bool(true) + when(io.sent_x_rep_ack) { state := s_busy } + } + is(s_busy) { // Nothing left to do but wait for transaction to complete + when (io.xact_finish) { + state := s_idle + } + } + } +} + +abstract class CoherenceHub(ntiles: Int) extends Component with CoherencePolicy { + val io = new Bundle { + val tiles = Vec(ntiles) { new ioTileLink() }.flip + val mem = new ioMem + } +} + +class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence +{ + val x_init = io.tiles(0).xact_init + val is_write = x_init.bits.t_type === X_INIT_WRITE_UNCACHED + x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp + io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) + io.mem.req_cmd.bits.rw := is_write + io.mem.req_cmd.bits.tag := x_init.bits.tile_xact_id + io.mem.req_cmd.bits.addr := x_init.bits.address + io.mem.req_data <> io.tiles(0).xact_init_data + + val x_rep = io.tiles(0).xact_rep + x_rep.bits.t_type := Mux(io.mem.resp.valid, X_REP_READ_EXCLUSIVE, X_REP_WRITE_UNCACHED) + x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) + x_rep.bits.global_xact_id := UFix(0) // don't care + x_rep.bits.data := io.mem.resp.bits.data + x_rep.bits.require_ack := Bool(true) + x_rep.valid := io.mem.resp.valid || x_init.valid && is_write && io.mem.req_cmd.ready + + io.tiles(0).xact_abort.valid := Bool(false) + io.tiles(0).xact_finish.ready := Bool(true) + io.tiles(0).probe_req.valid := Bool(false) + io.tiles(0).probe_rep.ready := Bool(true) + io.tiles(0).probe_rep_data.ready := Bool(true) +} + + +class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourStateCoherence +{ + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _)) + + val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } + val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } + val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } + val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + + val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } + val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } + val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } + val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bits(width = TILE_ID_BITS)} } + val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } + + for( i <- 0 until NGLOBAL_XACTS) { + val t = trackerList(i).io + busy_arr(i) := t.busy + addr_arr(i) := t.addr + init_tile_id_arr(i) := t.init_tile_id + tile_xact_id_arr(i) := t.tile_xact_id + t_type_arr(i) := t.t_type + sh_count_arr(i) := t.sharer_count + send_x_rep_ack_arr(i) := t.send_x_rep_ack + t.xact_finish := do_free_arr(i) + t.p_data.bits.tile_id := p_data_tile_id_arr(i) + t.p_data.valid := p_data_valid_arr(i) + t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits + t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits + t.sent_x_rep_ack := sent_x_rep_ack_arr(i) + do_free_arr(i) := Bool(false) + sent_x_rep_ack_arr(i) := Bool(false) + p_data_tile_id_arr(i) := Bits(0, width = TILE_ID_BITS) + p_data_valid_arr(i) := Bool(false) + for( j <- 0 until ntiles) { + p_rep_cnt_dec_arr(i)(j) := Bool(false) + p_req_cnt_inc_arr(i)(j) := Bool(false) + } + } + + val p_rep_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY + val x_init_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY + + // Free finished transactions + for( j <- 0 until ntiles ) { + val finish = io.tiles(j).xact_finish + when (finish.valid) { + do_free_arr(finish.bits.global_xact_id) := Bool(true) + } + finish.ready := Bool(true) + } + + // Reply to initial requestor + // Forward memory responses from mem to tile or arbitrate to ack + val mem_idx = io.mem.resp.bits.tag + val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) + for( j <- 0 until ntiles ) { + val rep = io.tiles(j).xact_rep + rep.bits.t_type := UFix(0) + rep.bits.tile_xact_id := UFix(0) + rep.bits.global_xact_id := UFix(0) + rep.bits.data := io.mem.resp.bits.data + rep.bits.require_ack := Bool(true) + rep.valid := Bool(false) + when(io.mem.resp.valid && (UFix(j) === init_tile_id_arr(mem_idx))) { + rep.bits.t_type := getTransactionReplyType(t_type_arr(mem_idx), sh_count_arr(mem_idx)) + rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) + rep.bits.global_xact_id := mem_idx + rep.valid := Bool(true) + } . otherwise { + rep.bits.t_type := getTransactionReplyType(t_type_arr(ack_idx), sh_count_arr(ack_idx)) + rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) + rep.bits.global_xact_id := ack_idx + when (UFix(j) === init_tile_id_arr(ack_idx)) { + rep.valid := send_x_rep_ack_arr.toBits.orR + sent_x_rep_ack_arr(ack_idx) := Bool(true) + } + } + } + // If there were a ready signal due to e.g. intervening network use: + //io.mem.resp.ready := io.tiles(init_tile_id_arr.read(mem_idx)).xact_rep.ready + + // Create an arbiter for the one memory port + // We have to arbitrate between the different trackers' memory requests + // and once we have picked a request, get the right write data + val mem_req_cmd_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemReqCmd() } + val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemData() } + for( i <- 0 until NGLOBAL_XACTS ) { + mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd + mem_req_cmd_arb.io.lock(i) <> trackerList(i).io.mem_req_lock + mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data + mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock + } + io.mem.req_cmd <> mem_req_cmd_arb.io.out + io.mem.req_data <> mem_req_data_arb.io.out + + // Handle probe replies, which may or may not have data + for( j <- 0 until ntiles ) { + val p_rep = io.tiles(j).probe_rep + val p_rep_data = io.tiles(j).probe_rep_data + val idx = p_rep.bits.global_xact_id + val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) + val do_pop = foldR(pop_p_reps)(_ || _) + p_rep.ready := Bool(true) + p_rep_data_dep_list(j).io.enq.valid := do_pop + p_rep_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_p_reps) + p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) + when (p_rep.valid && co.messageHasData(p_rep.bits)) { + p_data_valid_arr(idx) := Bool(true) + p_data_tile_id_arr(idx) := UFix(j) + } + p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) + } + for( i <- 0 until NGLOBAL_XACTS ) { + trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid + trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits + + trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) + trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) + + for( j <- 0 until ntiles) { + val p_rep = io.tiles(j).probe_rep + p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) + } + } + + // Nack conflicting transaction init attempts + val s_idle :: s_abort_drain :: s_abort_send :: s_abort_complete :: Nil = Enum(4){ UFix() } + val abort_state_arr = Vec(ntiles) { Reg(resetVal = s_idle) } + val want_to_abort_arr = Vec(ntiles) { Wire() { Bool()} } + for( j <- 0 until ntiles ) { + val x_init = io.tiles(j).xact_init + val x_init_data = io.tiles(j).xact_init_data + val x_abort = io.tiles(j).xact_abort + val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val conflicts = Vec(NGLOBAL_XACTS) { Wire() { Bool() } } + for( i <- 0 until NGLOBAL_XACTS) { + val t = trackerList(i).io + conflicts(i) := t.busy && x_init.valid && isCoherenceConflict(t.addr, x_init.bits.address) + } + x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id + want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && hasData(x_init.bits))) + + x_abort.valid := Bool(false) + switch(abort_state_arr(j)) { + is(s_idle) { + when(want_to_abort_arr(j)) { + when(hasData(x_init.bits)) { + abort_state_arr(j) := s_abort_drain + } . otherwise { + abort_state_arr(j) := s_abort_send + } + } + } + is(s_abort_drain) { // raises x_init_data.ready below + when(x_init_data.valid) { + abort_cnt := abort_cnt + UFix(1) + when(abort_cnt === ~UFix(0, width = log2up(REFILL_CYCLES))) { + abort_state_arr(j) := s_abort_send + } + } + } + is(s_abort_send) { // nothing is dequeued for now + x_abort.valid := Bool(true) + when(x_abort.ready) { + abort_state_arr(j) := s_abort_complete + } + } + is(s_abort_complete) { // raises x_init.ready below + abort_state_arr(j) := s_idle + } + } + } + + // Handle transaction initiation requests + // Only one allocation per cycle + // Init requests may or may not have data + val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } + val init_arb = (new Arbiter(ntiles)) { new TrackerAllocReq() } + for( i <- 0 until NGLOBAL_XACTS ) { + alloc_arb.io.in(i).valid := !trackerList(i).io.busy + trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready + trackerList(i).io.alloc_req.bits := init_arb.io.out.bits + trackerList(i).io.alloc_req.valid := init_arb.io.out.valid + + trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits + trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid + trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) + trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) + } + for( j <- 0 until ntiles ) { + val x_init = io.tiles(j).xact_init + val x_init_data = io.tiles(j).xact_init_data + val x_init_data_dep = x_init_data_dep_list(j).io.deq + init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid + init_arb.io.in(j).bits.xact_init := x_init.bits + init_arb.io.in(j).bits.tile_id := UFix(j) + val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) + val do_pop = foldR(pop_x_inits)(_||_) + x_init_data_dep_list(j).io.enq.valid := do_pop && hasData(x_init.bits) && (abort_state_arr(j) === s_idle) + x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) + x_init.ready := (abort_state_arr(j) === s_abort_complete) || do_pop + x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) + x_init_data_dep.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) + } + + alloc_arb.io.out.ready := init_arb.io.out.valid + + // Handle probe request generation + // Must arbitrate for each request port + val p_req_arb_arr = List.fill(ntiles)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) + for( j <- 0 until ntiles ) { + for( i <- 0 until NGLOBAL_XACTS ) { + val t = trackerList(i).io + p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits + p_req_arb_arr(j).io.in(i).valid := t.probe_req.valid && t.push_p_req(j) + p_req_cnt_inc_arr(i)(j) := p_req_arb_arr(j).io.in(i).ready + } + p_req_arb_arr(j).io.out <> io.tiles(j).probe_req + } + +} From d301336c3366bc18263ed486d4aa055b85a598ba Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 3 Apr 2012 12:03:05 -0700 Subject: [PATCH 075/374] Refactored coherence better from uncore hub, better coherence function names --- uncore/uncore.scala | 503 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 503 insertions(+) create mode 100644 uncore/uncore.scala diff --git a/uncore/uncore.scala b/uncore/uncore.scala new file mode 100644 index 00000000..fa6e2b1e --- /dev/null +++ b/uncore/uncore.scala @@ -0,0 +1,503 @@ +package rocket + +import Chisel._ +import Constants._ + +class MemData extends Bundle { + val data = Bits(width = MEM_DATA_BITS) +} + +class MemReqCmd() extends Bundle +{ + val rw = Bool() + val addr = UFix(width = PADDR_BITS - OFFSET_BITS) + val tag = Bits(width = MEM_TAG_BITS) +} + +class MemResp () extends MemData +{ + val tag = Bits(width = MEM_TAG_BITS) +} + +class ioMem() extends Bundle +{ + val req_cmd = (new ioDecoupled) { new MemReqCmd() } + val req_data = (new ioDecoupled) { new MemData() } + val resp = (new ioPipe) { new MemResp() }.flip +} + +class TrackerProbeData extends Bundle { + val tile_id = Bits(width = TILE_ID_BITS) +} + +class TrackerAllocReq extends Bundle { + val xact_init = new TransactionInit() + val tile_id = Bits(width = TILE_ID_BITS) +} + +class TrackerDependency extends Bundle { + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) +} + +class ioTileLink extends Bundle { + val xact_init = (new ioDecoupled) { new TransactionInit } + val xact_init_data = (new ioDecoupled) { new TransactionInitData } + val xact_abort = (new ioDecoupled) { new TransactionAbort }.flip + val probe_req = (new ioDecoupled) { new ProbeRequest }.flip + val probe_rep = (new ioDecoupled) { new ProbeReply } + val probe_rep_data = (new ioDecoupled) { new ProbeReplyData } + val xact_rep = (new ioPipe) { new TransactionReply }.flip + val xact_finish = (new ioDecoupled) { new TransactionFinish } +} + +class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherence { + val io = new Bundle { + val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip + val p_data = (new ioPipe) { new TrackerProbeData }.flip + val can_alloc = Bool(INPUT) + val xact_finish = Bool(INPUT) + val p_rep_cnt_dec = Bits(ntiles, INPUT) + val p_req_cnt_inc = Bits(ntiles, INPUT) + val p_rep_data = (new ioPipe) { new ProbeReplyData }.flip + val x_init_data = (new ioPipe) { new TransactionInitData }.flip + val sent_x_rep_ack = Bool(INPUT) + val p_rep_data_dep = (new ioPipe) { new TrackerDependency }.flip + val x_init_data_dep = (new ioPipe) { new TrackerDependency }.flip + + val mem_req_cmd = (new ioDecoupled) { new MemReqCmd } + val mem_req_data = (new ioDecoupled) { new MemData } + val mem_req_lock = Bool(OUTPUT) + val probe_req = (new ioDecoupled) { new ProbeRequest } + val busy = Bool(OUTPUT) + val addr = Bits(PADDR_BITS - OFFSET_BITS, OUTPUT) + val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) + val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) + val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) + val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) + val t_type = Bits(X_INIT_TYPE_BITS, OUTPUT) + val push_p_req = Bits(ntiles, OUTPUT) + val pop_p_rep = Bits(ntiles, OUTPUT) + val pop_p_rep_data = Bits(ntiles, OUTPUT) + val pop_p_rep_dep = Bits(ntiles, OUTPUT) + val pop_x_init = Bits(ntiles, OUTPUT) + val pop_x_init_data = Bits(ntiles, OUTPUT) + val pop_x_init_dep = Bits(ntiles, OUTPUT) + val send_x_rep_ack = Bool(OUTPUT) + } + + def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioPipe[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { + req_cmd.valid := !cmd_sent && at_front_of_dep_queue + req_cmd.bits.rw := Bool(true) + req_data.valid := data.valid && at_front_of_dep_queue + req_data.bits := data.bits + lock := at_front_of_dep_queue + when(req_cmd.ready && req_cmd.valid) { + cmd_sent := Bool(true) + } + when(req_data.ready && at_front_of_dep_queue) { + pop_data := UFix(1) << tile_id + when (data.valid) { + mem_cnt := mem_cnt_next + when(mem_cnt_next === UFix(0)) { + pop_dep := UFix(1) << tile_id + trigger := Bool(false) + } + } + } + } + + def doMemReqRead(req_cmd: ioDecoupled[MemReqCmd], trigger: Bool) { + req_cmd.valid := Bool(true) + req_cmd.bits.rw := Bool(false) + when(req_cmd.ready) { + trigger := Bool(false) + } + } + + val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } + val state = Reg(resetVal = s_idle) + val addr_ = Reg{ UFix() } + val t_type_ = Reg{ Bits() } + val init_tile_id_ = Reg{ Bits() } + val tile_xact_id_ = Reg{ Bits() } + val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2up(ntiles))) + val p_req_flags = Reg(resetVal = Bits(0, width = ntiles)) + val p_rep_tile_id_ = Reg{ Bits() } + val x_needs_read = Reg(resetVal = Bool(false)) + val x_init_data_needs_write = Reg(resetVal = Bool(false)) + val p_rep_data_needs_write = Reg(resetVal = Bool(false)) + val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val mem_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val mem_cnt_next = mem_cnt + UFix(1) + val mem_cnt_max = ~UFix(0, width = log2up(REFILL_CYCLES)) + + io.busy := state != s_idle + io.addr := addr_ + io.init_tile_id := init_tile_id_ + io.p_rep_tile_id := p_rep_tile_id_ + io.tile_xact_id := tile_xact_id_ + io.sharer_count := UFix(ntiles) // TODO: Broadcast only + io.t_type := t_type_ + + io.mem_req_cmd.valid := Bool(false) + io.mem_req_cmd.bits.rw := Bool(false) + io.mem_req_cmd.bits.addr := addr_ + io.mem_req_cmd.bits.tag := UFix(id) + io.mem_req_data.valid := Bool(false) + io.mem_req_data.bits.data := UFix(0) + io.mem_req_lock := Bool(false) + io.probe_req.valid := Bool(false) + io.probe_req.bits.p_type := getProbeRequestType(t_type_, UFix(0)) + io.probe_req.bits.global_xact_id := UFix(id) + io.probe_req.bits.address := addr_ + io.push_p_req := Bits(0, width = ntiles) + io.pop_p_rep := Bits(0, width = ntiles) + io.pop_p_rep_data := Bits(0, width = ntiles) + io.pop_p_rep_dep := Bits(0, width = ntiles) + io.pop_x_init := Bits(0, width = ntiles) + io.pop_x_init_data := Bits(0, width = ntiles) + io.pop_x_init_dep := Bits(0, width = ntiles) + io.send_x_rep_ack := Bool(false) + + switch (state) { + is(s_idle) { + when( io.alloc_req.valid && io.can_alloc ) { + addr_ := io.alloc_req.bits.xact_init.address + t_type_ := io.alloc_req.bits.xact_init.t_type + init_tile_id_ := io.alloc_req.bits.tile_id + tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id + x_init_data_needs_write := hasData(io.alloc_req.bits.xact_init) + x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) + if(ntiles > 1) p_rep_count := UFix(ntiles-1) + val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only + p_req_flags := p_req_initial_flags + mem_cnt := UFix(0) + p_w_mem_cmd_sent := Bool(false) + x_w_mem_cmd_sent := Bool(false) + io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id + state := Mux(p_req_initial_flags.orR, s_probe, s_mem) + } + } + is(s_probe) { + when(p_req_flags.orR) { + io.push_p_req := p_req_flags + io.probe_req.valid := Bool(true) + } + when(io.p_req_cnt_inc.orR) { + p_req_flags := p_req_flags & ~io.p_req_cnt_inc // unflag sent reqs + } + when(io.p_rep_cnt_dec.orR) { + val dec = PopCount(io.p_rep_cnt_dec) + io.pop_p_rep := io.p_rep_cnt_dec + if(ntiles > 1) p_rep_count := p_rep_count - dec + when(p_rep_count === dec) { + state := s_mem + } + } + when(io.p_data.valid) { + p_rep_data_needs_write := Bool(true) + p_rep_tile_id_ := io.p_data.bits.tile_id + } + } + is(s_mem) { + when (p_rep_data_needs_write) { + doMemReqWrite(io.mem_req_cmd, + io.mem_req_data, + io.mem_req_lock, + io.p_rep_data, + p_rep_data_needs_write, + p_w_mem_cmd_sent, + io.pop_p_rep_data, + io.pop_p_rep_dep, + io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id)), + p_rep_tile_id_) + } . elsewhen(x_init_data_needs_write) { + doMemReqWrite(io.mem_req_cmd, + io.mem_req_data, + io.mem_req_lock, + io.x_init_data, + x_init_data_needs_write, + x_w_mem_cmd_sent, + io.pop_x_init_data, + io.pop_x_init_dep, + io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id)), + init_tile_id_) + } . elsewhen (x_needs_read) { + doMemReqRead(io.mem_req_cmd, x_needs_read) + } . otherwise { + state := Mux(needsAckReply(t_type_, UFix(0)), s_ack, s_busy) + } + } + is(s_ack) { + io.send_x_rep_ack := Bool(true) + when(io.sent_x_rep_ack) { state := s_busy } + } + is(s_busy) { // Nothing left to do but wait for transaction to complete + when (io.xact_finish) { + state := s_idle + } + } + } +} + +abstract class CoherenceHub(ntiles: Int) extends Component with CoherencePolicy { + val io = new Bundle { + val tiles = Vec(ntiles) { new ioTileLink() }.flip + val mem = new ioMem + } +} + +class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence +{ + val x_init = io.tiles(0).xact_init + val is_write = x_init.bits.t_type === X_INIT_WRITE_UNCACHED + x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp + io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) + io.mem.req_cmd.bits.rw := is_write + io.mem.req_cmd.bits.tag := x_init.bits.tile_xact_id + io.mem.req_cmd.bits.addr := x_init.bits.address + io.mem.req_data <> io.tiles(0).xact_init_data + + val x_rep = io.tiles(0).xact_rep + x_rep.bits.t_type := Mux(io.mem.resp.valid, X_REP_READ_EXCLUSIVE, X_REP_WRITE_UNCACHED) + x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) + x_rep.bits.global_xact_id := UFix(0) // don't care + x_rep.bits.data := io.mem.resp.bits.data + x_rep.bits.require_ack := Bool(true) + x_rep.valid := io.mem.resp.valid || x_init.valid && is_write && io.mem.req_cmd.ready + + io.tiles(0).xact_abort.valid := Bool(false) + io.tiles(0).xact_finish.ready := Bool(true) + io.tiles(0).probe_req.valid := Bool(false) + io.tiles(0).probe_rep.ready := Bool(true) + io.tiles(0).probe_rep_data.ready := Bool(true) +} + + +class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourStateCoherence +{ + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _)) + + val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } + val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } + val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } + val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + + val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } + val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } + val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } + val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bits(width = TILE_ID_BITS)} } + val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } + + for( i <- 0 until NGLOBAL_XACTS) { + val t = trackerList(i).io + busy_arr(i) := t.busy + addr_arr(i) := t.addr + init_tile_id_arr(i) := t.init_tile_id + tile_xact_id_arr(i) := t.tile_xact_id + t_type_arr(i) := t.t_type + sh_count_arr(i) := t.sharer_count + send_x_rep_ack_arr(i) := t.send_x_rep_ack + t.xact_finish := do_free_arr(i) + t.p_data.bits.tile_id := p_data_tile_id_arr(i) + t.p_data.valid := p_data_valid_arr(i) + t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits + t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits + t.sent_x_rep_ack := sent_x_rep_ack_arr(i) + do_free_arr(i) := Bool(false) + sent_x_rep_ack_arr(i) := Bool(false) + p_data_tile_id_arr(i) := Bits(0, width = TILE_ID_BITS) + p_data_valid_arr(i) := Bool(false) + for( j <- 0 until ntiles) { + p_rep_cnt_dec_arr(i)(j) := Bool(false) + p_req_cnt_inc_arr(i)(j) := Bool(false) + } + } + + val p_rep_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY + val x_init_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY + + // Free finished transactions + for( j <- 0 until ntiles ) { + val finish = io.tiles(j).xact_finish + when (finish.valid) { + do_free_arr(finish.bits.global_xact_id) := Bool(true) + } + finish.ready := Bool(true) + } + + // Reply to initial requestor + // Forward memory responses from mem to tile or arbitrate to ack + val mem_idx = io.mem.resp.bits.tag + val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) + for( j <- 0 until ntiles ) { + val rep = io.tiles(j).xact_rep + rep.bits.t_type := UFix(0) + rep.bits.tile_xact_id := UFix(0) + rep.bits.global_xact_id := UFix(0) + rep.bits.data := io.mem.resp.bits.data + rep.bits.require_ack := Bool(true) + rep.valid := Bool(false) + when(io.mem.resp.valid && (UFix(j) === init_tile_id_arr(mem_idx))) { + rep.bits.t_type := getTransactionReplyType(t_type_arr(mem_idx), sh_count_arr(mem_idx)) + rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) + rep.bits.global_xact_id := mem_idx + rep.valid := Bool(true) + } . otherwise { + rep.bits.t_type := getTransactionReplyType(t_type_arr(ack_idx), sh_count_arr(ack_idx)) + rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) + rep.bits.global_xact_id := ack_idx + when (UFix(j) === init_tile_id_arr(ack_idx)) { + rep.valid := send_x_rep_ack_arr.toBits.orR + sent_x_rep_ack_arr(ack_idx) := Bool(true) + } + } + } + // If there were a ready signal due to e.g. intervening network use: + //io.mem.resp.ready := io.tiles(init_tile_id_arr.read(mem_idx)).xact_rep.ready + + // Create an arbiter for the one memory port + // We have to arbitrate between the different trackers' memory requests + // and once we have picked a request, get the right write data + val mem_req_cmd_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemReqCmd() } + val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemData() } + for( i <- 0 until NGLOBAL_XACTS ) { + mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd + mem_req_cmd_arb.io.lock(i) <> trackerList(i).io.mem_req_lock + mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data + mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock + } + io.mem.req_cmd <> mem_req_cmd_arb.io.out + io.mem.req_data <> mem_req_data_arb.io.out + + // Handle probe replies, which may or may not have data + for( j <- 0 until ntiles ) { + val p_rep = io.tiles(j).probe_rep + val p_rep_data = io.tiles(j).probe_rep_data + val idx = p_rep.bits.global_xact_id + val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) + val do_pop = foldR(pop_p_reps)(_ || _) + p_rep.ready := Bool(true) + p_rep_data_dep_list(j).io.enq.valid := do_pop + p_rep_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_p_reps) + p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) + when (p_rep.valid && co.messageHasData(p_rep.bits)) { + p_data_valid_arr(idx) := Bool(true) + p_data_tile_id_arr(idx) := UFix(j) + } + p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) + } + for( i <- 0 until NGLOBAL_XACTS ) { + trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid + trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits + + trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) + trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) + + for( j <- 0 until ntiles) { + val p_rep = io.tiles(j).probe_rep + p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) + } + } + + // Nack conflicting transaction init attempts + val s_idle :: s_abort_drain :: s_abort_send :: s_abort_complete :: Nil = Enum(4){ UFix() } + val abort_state_arr = Vec(ntiles) { Reg(resetVal = s_idle) } + val want_to_abort_arr = Vec(ntiles) { Wire() { Bool()} } + for( j <- 0 until ntiles ) { + val x_init = io.tiles(j).xact_init + val x_init_data = io.tiles(j).xact_init_data + val x_abort = io.tiles(j).xact_abort + val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val conflicts = Vec(NGLOBAL_XACTS) { Wire() { Bool() } } + for( i <- 0 until NGLOBAL_XACTS) { + val t = trackerList(i).io + conflicts(i) := t.busy && x_init.valid && isCoherenceConflict(t.addr, x_init.bits.address) + } + x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id + want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && hasData(x_init.bits))) + + x_abort.valid := Bool(false) + switch(abort_state_arr(j)) { + is(s_idle) { + when(want_to_abort_arr(j)) { + when(hasData(x_init.bits)) { + abort_state_arr(j) := s_abort_drain + } . otherwise { + abort_state_arr(j) := s_abort_send + } + } + } + is(s_abort_drain) { // raises x_init_data.ready below + when(x_init_data.valid) { + abort_cnt := abort_cnt + UFix(1) + when(abort_cnt === ~UFix(0, width = log2up(REFILL_CYCLES))) { + abort_state_arr(j) := s_abort_send + } + } + } + is(s_abort_send) { // nothing is dequeued for now + x_abort.valid := Bool(true) + when(x_abort.ready) { + abort_state_arr(j) := s_abort_complete + } + } + is(s_abort_complete) { // raises x_init.ready below + abort_state_arr(j) := s_idle + } + } + } + + // Handle transaction initiation requests + // Only one allocation per cycle + // Init requests may or may not have data + val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } + val init_arb = (new Arbiter(ntiles)) { new TrackerAllocReq() } + for( i <- 0 until NGLOBAL_XACTS ) { + alloc_arb.io.in(i).valid := !trackerList(i).io.busy + trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready + trackerList(i).io.alloc_req.bits := init_arb.io.out.bits + trackerList(i).io.alloc_req.valid := init_arb.io.out.valid + + trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits + trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid + trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) + trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) + } + for( j <- 0 until ntiles ) { + val x_init = io.tiles(j).xact_init + val x_init_data = io.tiles(j).xact_init_data + val x_init_data_dep = x_init_data_dep_list(j).io.deq + init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid + init_arb.io.in(j).bits.xact_init := x_init.bits + init_arb.io.in(j).bits.tile_id := UFix(j) + val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) + val do_pop = foldR(pop_x_inits)(_||_) + x_init_data_dep_list(j).io.enq.valid := do_pop && hasData(x_init.bits) && (abort_state_arr(j) === s_idle) + x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) + x_init.ready := (abort_state_arr(j) === s_abort_complete) || do_pop + x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) + x_init_data_dep.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) + } + + alloc_arb.io.out.ready := init_arb.io.out.valid + + // Handle probe request generation + // Must arbitrate for each request port + val p_req_arb_arr = List.fill(ntiles)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) + for( j <- 0 until ntiles ) { + for( i <- 0 until NGLOBAL_XACTS ) { + val t = trackerList(i).io + p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits + p_req_arb_arr(j).io.in(i).valid := t.probe_req.valid && t.push_p_req(j) + p_req_cnt_inc_arr(i)(j) := p_req_arb_arr(j).io.in(i).ready + } + p_req_arb_arr(j).io.out <> io.tiles(j).probe_req + } + +} From 6bc47a55b42e28b34cf4845a82387f33cc3c91e9 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 3 Apr 2012 18:06:02 -0700 Subject: [PATCH 076/374] changed coherence message type names --- uncore/coherence.scala | 147 ++++++++++++++++++----------------------- uncore/uncore.scala | 28 ++++---- 2 files changed, 78 insertions(+), 97 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index ee51e469..7af9c903 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -4,7 +4,7 @@ import Chisel._ import Constants._ class TransactionInit extends Bundle { - val t_type = Bits(width = X_INIT_TYPE_BITS) + val x_type = Bits(width = X_INIT_TYPE_BITS) val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) val address = UFix(width = PADDR_BITS - OFFSET_BITS) } @@ -29,7 +29,7 @@ class ProbeReply extends Bundle { class ProbeReplyData extends MemData class TransactionReply extends MemData { - val t_type = Bits(width = X_REP_TYPE_BITS) + val x_type = Bits(width = X_REP_TYPE_BITS) val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) val require_ack = Bool() @@ -54,13 +54,13 @@ trait CoherencePolicy { } trait ThreeStateIncoherence extends CoherencePolicy { val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } - val X_INIT_READ_SHARED = UFix(0, 2) - val X_INIT_READ_EXCLUSIVE = UFix(1, 2) - val X_INIT_WRITE_UNCACHED = UFix(3, 2) - val X_REP_READ_SHARED = UFix(0, X_REP_TYPE_BITS) - val X_REP_READ_EXCLUSIVE = UFix(1, X_REP_TYPE_BITS) - val X_REP_WRITE_UNCACHED = UFix(3, X_REP_TYPE_BITS) - val P_REP_INVALIDATE_ACK = UFix(3, P_REP_TYPE_BITS) + val xactInitReadShared = UFix(0, 2) + val xactInitReadExclusive = UFix(1, 2) + val xactInitWriteUncached = UFix(3, 2) + val xactReplyReadShared = UFix(0, X_REP_TYPE_BITS) + val xactReplyReadExclusive = UFix(1, X_REP_TYPE_BITS) + val xactReplyWriteUncached = UFix(3, X_REP_TYPE_BITS) + val probeRepInvalidateAck = UFix(3, P_REP_TYPE_BITS) def isHit ( cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) @@ -84,57 +84,38 @@ trait ThreeStateIncoherence extends CoherencePolicy { def newStateOnHit(cmd: Bits, state: UFix): UFix = newState(cmd, state) def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) + Mux(write || cmd === M_PFW, xactInitReadExclusive, xactInitReadShared) } def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, X_INIT_READ_EXCLUSIVE, outstanding.t_type) + Mux(write, xactInitReadExclusive, outstanding.x_type) } def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = Bool(false) def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { - Mux(outstanding.t_type === X_INIT_READ_EXCLUSIVE, tileDirty, tileClean) + Mux(outstanding.x_type === xactInitReadExclusive, tileDirty, tileClean) } def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = state def newProbeReply (incoming: ProbeRequest, has_data: Bool): ProbeReply = { val reply = Wire() { new ProbeReply() } - reply.p_type := P_REP_INVALIDATE_ACK + reply.p_type := probeRepInvalidateAck reply.global_xact_id := UFix(0) reply } def probeReplyHasData (reply: ProbeReply): Bool = Bool(false) - def transactionInitHasData (init: TransactionInit): Bool = (init.t_type === X_INIT_WRITE_UNCACHED) + def transactionInitHasData (init: TransactionInit): Bool = (init.x_type === xactInitWriteUncached) } trait FourStateCoherence extends CoherencePolicy { val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(4){ UFix() } val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(3){ UFix() } - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } - val X_INIT_READ_SHARED = UFix(0, X_INIT_TYPE_BITS) - val X_INIT_READ_EXCLUSIVE = UFix(1, X_INIT_TYPE_BITS) - val X_INIT_READ_UNCACHED = UFix(2, X_INIT_TYPE_BITS) - val X_INIT_WRITE_UNCACHED = UFix(3, X_INIT_TYPE_BITS) + val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: Nil = Enum(4){ UFix() } + val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: Nil = Enum(5){ UFix() } + val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() } + val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() } - val X_REP_READ_SHARED = UFix(0, X_REP_TYPE_BITS) - val X_REP_READ_EXCLUSIVE = UFix(1, X_REP_TYPE_BITS) - val X_REP_READ_UNCACHED = UFix(2, X_REP_TYPE_BITS) - val X_REP_WRITE_UNCACHED = UFix(3, X_REP_TYPE_BITS) - val X_REP_READ_EXCLUSIVE_ACK = UFix(4, X_REP_TYPE_BITS) - - val P_REQ_INVALIDATE = UFix(0, P_REQ_TYPE_BITS) - val P_REQ_DOWNGRADE = UFix(1, P_REQ_TYPE_BITS) - val P_REQ_COPY = UFix(2, P_REQ_TYPE_BITS) - - val P_REP_INVALIDATE_DATA = UFix(0, P_REP_TYPE_BITS) - val P_REP_DOWNGRADE_DATA = UFix(1, P_REP_TYPE_BITS) - val P_REP_COPY_DATA = UFix(2, P_REP_TYPE_BITS) - val P_REP_INVALIDATE_ACK = UFix(3, P_REP_TYPE_BITS) - val P_REP_DOWNGRADE_ACK = UFix(4, P_REP_TYPE_BITS) - val P_REP_COPY_ACK = UFix(5, P_REP_TYPE_BITS) - - - def isHit ( cmd: Bits, state: UFix): Bool = { + def isHit (cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) Mux(write, (state === tileExclusiveClean || state === tileExclusiveDirty), (state === tileShared || state === tileExclusiveClean || state === tileExclusiveDirty)) @@ -145,8 +126,8 @@ trait FourStateCoherence extends CoherencePolicy { def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { val (read, write) = cpuCmdToRW(cmd) - (read && (outstanding.t_type === X_INIT_READ_UNCACHED || outstanding.t_type === X_INIT_WRITE_UNCACHED)) || - (write && (outstanding.t_type != X_INIT_READ_EXCLUSIVE)) + (read && (outstanding.x_type === xactInitReadUncached || outstanding.x_type === xactInitWriteUncached)) || + (write && (outstanding.x_type != xactInitReadExclusive)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( @@ -171,44 +152,44 @@ trait FourStateCoherence extends CoherencePolicy { def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { - MuxLookup(incoming.t_type, tileInvalid, Array( - X_REP_READ_SHARED -> tileShared, - X_REP_READ_EXCLUSIVE -> Mux(outstanding.t_type === X_INIT_READ_EXCLUSIVE, tileExclusiveDirty, tileExclusiveClean), - X_REP_READ_EXCLUSIVE_ACK -> tileExclusiveDirty, - X_REP_READ_UNCACHED -> tileInvalid, - X_REP_WRITE_UNCACHED -> tileInvalid + MuxLookup(incoming.x_type, tileInvalid, Array( + xactReplyReadShared -> tileShared, + xactReplyReadExclusive -> Mux(outstanding.x_type === xactInitReadExclusive, tileExclusiveDirty, tileExclusiveClean), + xactReplyReadExclusiveAck -> tileExclusiveDirty, + xactReplyReadUncached -> tileInvalid, + xactReplyWriteUncached -> tileInvalid )) } def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { MuxLookup(incoming.p_type, state, Array( - probeInvalidate -> tileInvalid, - probeDowngrade -> tileShared, - probeCopy -> state + probeReqInvalidate -> tileInvalid, + probeReqDowngrade -> tileShared, + probeReqCopy -> state )) } def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) + Mux(write || cmd === M_PFW, xactInitReadExclusive, xactInitReadShared) } def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, X_INIT_READ_EXCLUSIVE, outstanding.t_type) + Mux(write, xactInitReadExclusive, outstanding.x_type) } - def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = X_INIT_WRITE_UNCACHED + def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { val reply = Wire() { new ProbeReply() } - val with_data = MuxLookup(incoming.p_type, P_REP_INVALIDATE_DATA, Array( - probeInvalidate -> P_REP_INVALIDATE_DATA, - probeDowngrade -> P_REP_DOWNGRADE_DATA, - probeCopy -> P_REP_COPY_DATA + val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( + probeReqInvalidate -> probeRepInvalidateData, + probeReqDowngrade -> probeRepDowngradeData, + probeReqCopy -> probeRepCopyData )) - val without_data = MuxLookup(incoming.p_type, P_REP_INVALIDATE_ACK, Array( - probeInvalidate -> P_REP_INVALIDATE_ACK, - probeDowngrade -> P_REP_DOWNGRADE_ACK, - probeCopy -> P_REP_COPY_ACK + val without_data = MuxLookup(incoming.p_type, probeRepInvalidateAck, Array( + probeReqInvalidate -> probeRepInvalidateAck, + probeReqDowngrade -> probeRepDowngradeAck, + probeReqCopy -> probeRepCopyAck )) reply.p_type := Mux(needsWriteback(state), with_data, without_data) reply.global_xact_id := incoming.global_xact_id @@ -216,44 +197,44 @@ trait FourStateCoherence extends CoherencePolicy { } def hasData (reply: ProbeReply): Bool = { - (reply.p_type === P_REP_INVALIDATE_DATA || - reply.p_type === P_REP_DOWNGRADE_DATA || - reply.p_type === P_REP_COPY_DATA) + (reply.p_type === probeRepInvalidateData || + reply.p_type === probeRepDowngradeData || + reply.p_type === probeRepCopyData) } def hasData (init: TransactionInit): Bool = { - (init.t_type === X_INIT_WRITE_UNCACHED) + (init.x_type === xactInitWriteUncached) } def hasData (reply: TransactionReply): Bool = { - (reply.t_type != X_REP_WRITE_UNCACHED && reply.t_type != X_REP_READ_EXCLUSIVE_ACK) + (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck) } def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getTransactionReplyType(t_type: UFix, count: UFix): Bits = { - MuxLookup(t_type, X_REP_READ_UNCACHED, Array( - X_INIT_READ_SHARED -> Mux(count > UFix(0), X_REP_READ_SHARED, X_REP_READ_EXCLUSIVE), - X_INIT_READ_EXCLUSIVE -> X_REP_READ_EXCLUSIVE, - X_INIT_READ_UNCACHED -> X_REP_READ_UNCACHED, - X_INIT_WRITE_UNCACHED -> X_REP_WRITE_UNCACHED + def getTransactionReplyType(x_type: UFix, count: UFix): Bits = { + MuxLookup(x_type, xactReplyReadUncached, Array( + xactInitReadShared -> Mux(count > UFix(0), xactReplyReadShared, xactReplyReadExclusive), + xactInitReadExclusive -> xactReplyReadExclusive, + xactInitReadUncached -> xactReplyReadUncached, + xactInitWriteUncached -> xactReplyWriteUncached )) } - def getProbeRequestType(t_type: UFix, global_state: UFix): UFix = { - MuxLookup(t_type, P_REQ_COPY, Array( - X_INIT_READ_SHARED -> P_REQ_DOWNGRADE, - X_INIT_READ_EXCLUSIVE -> P_REQ_INVALIDATE, - X_INIT_READ_UNCACHED -> P_REQ_COPY, - X_INIT_WRITE_UNCACHED -> P_REQ_INVALIDATE + def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = { + MuxLookup(x_type, probeReqCopy, Array( + xactInitReadShared -> probeReqDowngrade, + xactInitReadExclusive -> probeReqInvalidate, + xactInitReadUncached -> probeReqCopy, + xactInitWriteUncached -> probeReqInvalidate )) } - def needsMemRead(t_type: UFix, global_state: UFix): Bool = { - (t_type != X_INIT_WRITE_UNCACHED) + def needsMemRead(x_type: UFix, global_state: UFix): Bool = { + (x_type != xactInitWriteUncached) } - def needsMemWrite(t_type: UFix, global_state: UFix): Bool = { - (t_type === X_INIT_WRITE_UNCACHED) + def needsMemWrite(x_type: UFix, global_state: UFix): Bool = { + (x_type === xactInitWriteUncached) } - def needsAckReply(t_type: UFix, global_state: UFix): Bool = { - (t_type === X_INIT_WRITE_UNCACHED) + def needsAckReply(x_type: UFix, global_state: UFix): Bool = { + (x_type === xactInitWriteUncached) } } diff --git a/uncore/uncore.scala b/uncore/uncore.scala index fa6e2b1e..8dc88d44 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -74,7 +74,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) - val t_type = Bits(X_INIT_TYPE_BITS, OUTPUT) + val x_type = Bits(X_INIT_TYPE_BITS, OUTPUT) val push_p_req = Bits(ntiles, OUTPUT) val pop_p_rep = Bits(ntiles, OUTPUT) val pop_p_rep_data = Bits(ntiles, OUTPUT) @@ -117,7 +117,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) val addr_ = Reg{ UFix() } - val t_type_ = Reg{ Bits() } + val x_type_ = Reg{ Bits() } val init_tile_id_ = Reg{ Bits() } val tile_xact_id_ = Reg{ Bits() } val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2up(ntiles))) @@ -138,7 +138,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc io.p_rep_tile_id := p_rep_tile_id_ io.tile_xact_id := tile_xact_id_ io.sharer_count := UFix(ntiles) // TODO: Broadcast only - io.t_type := t_type_ + io.x_type := x_type_ io.mem_req_cmd.valid := Bool(false) io.mem_req_cmd.bits.rw := Bool(false) @@ -148,7 +148,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc io.mem_req_data.bits.data := UFix(0) io.mem_req_lock := Bool(false) io.probe_req.valid := Bool(false) - io.probe_req.bits.p_type := getProbeRequestType(t_type_, UFix(0)) + io.probe_req.bits.p_type := getProbeRequestType(x_type_, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) io.probe_req.bits.address := addr_ io.push_p_req := Bits(0, width = ntiles) @@ -164,11 +164,11 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc is(s_idle) { when( io.alloc_req.valid && io.can_alloc ) { addr_ := io.alloc_req.bits.xact_init.address - t_type_ := io.alloc_req.bits.xact_init.t_type + x_type_ := io.alloc_req.bits.xact_init.x_type init_tile_id_ := io.alloc_req.bits.tile_id tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := hasData(io.alloc_req.bits.xact_init) - x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) + x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) if(ntiles > 1) p_rep_count := UFix(ntiles-1) val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only p_req_flags := p_req_initial_flags @@ -226,7 +226,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { - state := Mux(needsAckReply(t_type_, UFix(0)), s_ack, s_busy) + state := Mux(needsAckReply(x_type_, UFix(0)), s_ack, s_busy) } } is(s_ack) { @@ -251,7 +251,7 @@ abstract class CoherenceHub(ntiles: Int) extends Component with CoherencePolicy class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence { val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.t_type === X_INIT_WRITE_UNCACHED + val is_write = x_init.bits.x_type === xactInitWriteUncached x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write @@ -260,7 +260,7 @@ class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence io.mem.req_data <> io.tiles(0).xact_init_data val x_rep = io.tiles(0).xact_rep - x_rep.bits.t_type := Mux(io.mem.resp.valid, X_REP_READ_EXCLUSIVE, X_REP_WRITE_UNCACHED) + x_rep.bits.x_type := Mux(io.mem.resp.valid, xactReplyReadExclusive, xactReplyWriteUncached) x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.bits.data @@ -283,7 +283,7 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } + val x_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } @@ -300,7 +300,7 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS addr_arr(i) := t.addr init_tile_id_arr(i) := t.init_tile_id tile_xact_id_arr(i) := t.tile_xact_id - t_type_arr(i) := t.t_type + x_type_arr(i) := t.x_type sh_count_arr(i) := t.sharer_count send_x_rep_ack_arr(i) := t.send_x_rep_ack t.xact_finish := do_free_arr(i) @@ -337,19 +337,19 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) for( j <- 0 until ntiles ) { val rep = io.tiles(j).xact_rep - rep.bits.t_type := UFix(0) + rep.bits.x_type := UFix(0) rep.bits.tile_xact_id := UFix(0) rep.bits.global_xact_id := UFix(0) rep.bits.data := io.mem.resp.bits.data rep.bits.require_ack := Bool(true) rep.valid := Bool(false) when(io.mem.resp.valid && (UFix(j) === init_tile_id_arr(mem_idx))) { - rep.bits.t_type := getTransactionReplyType(t_type_arr(mem_idx), sh_count_arr(mem_idx)) + rep.bits.x_type := getTransactionReplyType(x_type_arr(mem_idx), sh_count_arr(mem_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) rep.bits.global_xact_id := mem_idx rep.valid := Bool(true) } . otherwise { - rep.bits.t_type := getTransactionReplyType(t_type_arr(ack_idx), sh_count_arr(ack_idx)) + rep.bits.x_type := getTransactionReplyType(x_type_arr(ack_idx), sh_count_arr(ack_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) rep.bits.global_xact_id := ack_idx when (UFix(j) === init_tile_id_arr(ack_idx)) { From f7307ee41164fe47fee1a1730c23f8c2929f0c0f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 3 Apr 2012 18:06:02 -0700 Subject: [PATCH 077/374] changed coherence message type names --- uncore/uncore.scala | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index fa6e2b1e..8dc88d44 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -74,7 +74,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) - val t_type = Bits(X_INIT_TYPE_BITS, OUTPUT) + val x_type = Bits(X_INIT_TYPE_BITS, OUTPUT) val push_p_req = Bits(ntiles, OUTPUT) val pop_p_rep = Bits(ntiles, OUTPUT) val pop_p_rep_data = Bits(ntiles, OUTPUT) @@ -117,7 +117,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) val addr_ = Reg{ UFix() } - val t_type_ = Reg{ Bits() } + val x_type_ = Reg{ Bits() } val init_tile_id_ = Reg{ Bits() } val tile_xact_id_ = Reg{ Bits() } val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2up(ntiles))) @@ -138,7 +138,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc io.p_rep_tile_id := p_rep_tile_id_ io.tile_xact_id := tile_xact_id_ io.sharer_count := UFix(ntiles) // TODO: Broadcast only - io.t_type := t_type_ + io.x_type := x_type_ io.mem_req_cmd.valid := Bool(false) io.mem_req_cmd.bits.rw := Bool(false) @@ -148,7 +148,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc io.mem_req_data.bits.data := UFix(0) io.mem_req_lock := Bool(false) io.probe_req.valid := Bool(false) - io.probe_req.bits.p_type := getProbeRequestType(t_type_, UFix(0)) + io.probe_req.bits.p_type := getProbeRequestType(x_type_, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) io.probe_req.bits.address := addr_ io.push_p_req := Bits(0, width = ntiles) @@ -164,11 +164,11 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc is(s_idle) { when( io.alloc_req.valid && io.can_alloc ) { addr_ := io.alloc_req.bits.xact_init.address - t_type_ := io.alloc_req.bits.xact_init.t_type + x_type_ := io.alloc_req.bits.xact_init.x_type init_tile_id_ := io.alloc_req.bits.tile_id tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := hasData(io.alloc_req.bits.xact_init) - x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) + x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) if(ntiles > 1) p_rep_count := UFix(ntiles-1) val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only p_req_flags := p_req_initial_flags @@ -226,7 +226,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { - state := Mux(needsAckReply(t_type_, UFix(0)), s_ack, s_busy) + state := Mux(needsAckReply(x_type_, UFix(0)), s_ack, s_busy) } } is(s_ack) { @@ -251,7 +251,7 @@ abstract class CoherenceHub(ntiles: Int) extends Component with CoherencePolicy class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence { val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.t_type === X_INIT_WRITE_UNCACHED + val is_write = x_init.bits.x_type === xactInitWriteUncached x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write @@ -260,7 +260,7 @@ class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence io.mem.req_data <> io.tiles(0).xact_init_data val x_rep = io.tiles(0).xact_rep - x_rep.bits.t_type := Mux(io.mem.resp.valid, X_REP_READ_EXCLUSIVE, X_REP_WRITE_UNCACHED) + x_rep.bits.x_type := Mux(io.mem.resp.valid, xactReplyReadExclusive, xactReplyWriteUncached) x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.bits.data @@ -283,7 +283,7 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } + val x_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } @@ -300,7 +300,7 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS addr_arr(i) := t.addr init_tile_id_arr(i) := t.init_tile_id tile_xact_id_arr(i) := t.tile_xact_id - t_type_arr(i) := t.t_type + x_type_arr(i) := t.x_type sh_count_arr(i) := t.sharer_count send_x_rep_ack_arr(i) := t.send_x_rep_ack t.xact_finish := do_free_arr(i) @@ -337,19 +337,19 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) for( j <- 0 until ntiles ) { val rep = io.tiles(j).xact_rep - rep.bits.t_type := UFix(0) + rep.bits.x_type := UFix(0) rep.bits.tile_xact_id := UFix(0) rep.bits.global_xact_id := UFix(0) rep.bits.data := io.mem.resp.bits.data rep.bits.require_ack := Bool(true) rep.valid := Bool(false) when(io.mem.resp.valid && (UFix(j) === init_tile_id_arr(mem_idx))) { - rep.bits.t_type := getTransactionReplyType(t_type_arr(mem_idx), sh_count_arr(mem_idx)) + rep.bits.x_type := getTransactionReplyType(x_type_arr(mem_idx), sh_count_arr(mem_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) rep.bits.global_xact_id := mem_idx rep.valid := Bool(true) } . otherwise { - rep.bits.t_type := getTransactionReplyType(t_type_arr(ack_idx), sh_count_arr(ack_idx)) + rep.bits.x_type := getTransactionReplyType(x_type_arr(ack_idx), sh_count_arr(ack_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) rep.bits.global_xact_id := ack_idx when (UFix(j) === init_tile_id_arr(ack_idx)) { From 17a5d26c1ed11bd1f8adc09a3bf011f0dbf1785c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 4 Apr 2012 13:57:08 -0700 Subject: [PATCH 078/374] changed coherence type width names to represent max sizes for all protocols --- uncore/coherence.scala | 16 ++++++++-------- uncore/uncore.scala | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 7af9c903..be6638aa 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -4,7 +4,7 @@ import Chisel._ import Constants._ class TransactionInit extends Bundle { - val x_type = Bits(width = X_INIT_TYPE_BITS) + val x_type = Bits(width = X_INIT_TYPE_MAX_BITS) val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) val address = UFix(width = PADDR_BITS - OFFSET_BITS) } @@ -16,20 +16,20 @@ class TransactionAbort extends Bundle { } class ProbeRequest extends Bundle { - val p_type = Bits(width = P_REQ_TYPE_BITS) + val p_type = Bits(width = P_REQ_TYPE_MAX_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) val address = Bits(width = PADDR_BITS - OFFSET_BITS) } class ProbeReply extends Bundle { - val p_type = Bits(width = P_REP_TYPE_BITS) + val p_type = Bits(width = P_REP_TYPE_MAX_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } class ProbeReplyData extends MemData class TransactionReply extends MemData { - val x_type = Bits(width = X_REP_TYPE_BITS) + val x_type = Bits(width = X_REP_TYPE_MAX_BITS) val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) val require_ack = Bool() @@ -57,10 +57,10 @@ trait ThreeStateIncoherence extends CoherencePolicy { val xactInitReadShared = UFix(0, 2) val xactInitReadExclusive = UFix(1, 2) val xactInitWriteUncached = UFix(3, 2) - val xactReplyReadShared = UFix(0, X_REP_TYPE_BITS) - val xactReplyReadExclusive = UFix(1, X_REP_TYPE_BITS) - val xactReplyWriteUncached = UFix(3, X_REP_TYPE_BITS) - val probeRepInvalidateAck = UFix(3, P_REP_TYPE_BITS) + val xactReplyReadShared = UFix(0, X_REP_TYPE_MAX_BITS) + val xactReplyReadExclusive = UFix(1, X_REP_TYPE_MAX_BITS) + val xactReplyWriteUncached = UFix(3, X_REP_TYPE_MAX_BITS) + val probeRepInvalidateAck = UFix(3, P_REP_TYPE_MAX_BITS) def isHit ( cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 8dc88d44..b907c51d 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -74,7 +74,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) - val x_type = Bits(X_INIT_TYPE_BITS, OUTPUT) + val x_type = Bits(X_INIT_TYPE_MAX_BITS, OUTPUT) val push_p_req = Bits(ntiles, OUTPUT) val pop_p_rep = Bits(ntiles, OUTPUT) val pop_p_rep_data = Bits(ntiles, OUTPUT) @@ -283,7 +283,7 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val x_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } + val x_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_MAX_BITS)} } val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } From a68f5e016d5a9567095750909582e5b5e2a98153 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 4 Apr 2012 13:57:08 -0700 Subject: [PATCH 079/374] changed coherence type width names to represent max sizes for all protocols --- uncore/uncore.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 8dc88d44..b907c51d 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -74,7 +74,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) - val x_type = Bits(X_INIT_TYPE_BITS, OUTPUT) + val x_type = Bits(X_INIT_TYPE_MAX_BITS, OUTPUT) val push_p_req = Bits(ntiles, OUTPUT) val pop_p_rep = Bits(ntiles, OUTPUT) val pop_p_rep_data = Bits(ntiles, OUTPUT) @@ -283,7 +283,7 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val x_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } + val x_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_MAX_BITS)} } val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } From e71e3ce38f16b1b47e386807d0bf8f99b82fa372 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 4 Apr 2012 15:51:33 -0700 Subject: [PATCH 080/374] defined abstract coherence traits in base trait, added Incoherent trait, cleaned up incoherent policy --- uncore/coherence.scala | 114 ++++++++++++++++++++++++++++------------- uncore/uncore.scala | 4 +- 2 files changed, 79 insertions(+), 39 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index be6638aa..1bd19bbd 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -50,59 +50,99 @@ object cpuCmdToRW { } } -trait CoherencePolicy { } +trait CoherencePolicy { + def isHit (cmd: Bits, state: UFix): Bool + def isValid (state: UFix): Bool -trait ThreeStateIncoherence extends CoherencePolicy { + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool + def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool + def needsWriteback (state: UFix): Bool + + def newStateOnHit(cmd: Bits, state: UFix): UFix + def newStateOnCacheControl(cmd: Bits): UFix + def newStateOnWriteback(): UFix + def newStateOnFlush(): UFix + def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix + def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits + + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix + def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix + def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits + def getTransactionInitTypeOnWriteback(): Bits + + def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply + + def hasData (reply: ProbeReply): Bool + def hasData (init: TransactionInit): Bool + def hasData (reply: TransactionReply): Bool + + def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool + def getTransactionReplyType(x_type: UFix, count: UFix): Bits + def getProbeRequestType(x_type: UFix, global_state: UFix): UFix + def needsMemRead(x_type: UFix, global_state: UFix): Bool + def needsMemWrite(x_type: UFix, global_state: UFix): Bool + def needsAckReply(x_type: UFix, global_state: UFix): Bool +} + +trait IncoherentPolicy extends CoherencePolicy { + // UNIMPLEMENTED + def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = state + def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { + val reply = Wire() { new ProbeReply() } + reply.p_type := UFix(0) + reply.global_xact_id := UFix(0) + reply + } + def hasData (reply: ProbeReply) = Bool(false) + def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = Bool(false) + def getTransactionReplyType(x_type: UFix, count: UFix): Bits = Bits(0) + def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = UFix(0) + def needsMemRead(x_type: UFix, global_state: UFix): Bool = Bool(false) + def needsMemWrite(x_type: UFix, global_state: UFix): Bool = Bool(false) + def needsAckReply(x_type: UFix, global_state: UFix): Bool = Bool(false) +} + +trait ThreeStateIncoherence extends IncoherentPolicy { val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } - val xactInitReadShared = UFix(0, 2) - val xactInitReadExclusive = UFix(1, 2) - val xactInitWriteUncached = UFix(3, 2) - val xactReplyReadShared = UFix(0, X_REP_TYPE_MAX_BITS) - val xactReplyReadExclusive = UFix(1, X_REP_TYPE_MAX_BITS) - val xactReplyWriteUncached = UFix(3, X_REP_TYPE_MAX_BITS) - val probeRepInvalidateAck = UFix(3, P_REP_TYPE_MAX_BITS) + val xactInitReadClean :: xactInitReadDirty :: xactInitWriteback :: Nil = Enum(3){ UFix() } + val xactReplyData :: xactReplyAck :: Nil = Enum(2){ UFix() } + val probeRepInvalidateAck :: Nil = Enum(1){ UFix() } - def isHit ( cmd: Bits, state: UFix): Bool = { - val (read, write) = cpuCmdToRW(cmd) - ( state === tileClean || state === tileDirty) - } + def isHit ( cmd: Bits, state: UFix): Bool = (state === tileClean || state === tileDirty) + def isValid (state: UFix): Bool = state != tileInvalid - def isValid (state: UFix): Bool = { - state != tileInvalid - } + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit) = Bool(false) + def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = state === tileDirty + def needsWriteback (state: UFix): Bool = state === tileDirty - def needsWriteback (state: UFix): Bool = { - state === tileDirty - } - - def newStateOnWriteback() = tileInvalid - def newStateOnCacheControl(cmd: Bits) = tileInvalid def newState(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) Mux(write, tileDirty, Mux(read, Mux(state === tileDirty, tileDirty, tileClean), state)) } def newStateOnHit(cmd: Bits, state: UFix): UFix = newState(cmd, state) + def newStateOnCacheControl(cmd: Bits) = tileInvalid //TODO + def newStateOnWriteback() = tileInvalid + def newStateOnFlush() = tileInvalid + def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit) = { + MuxLookup(incoming.x_type, tileInvalid, Array( + xactReplyData -> Mux(outstanding.x_type === xactInitReadDirty, tileDirty, tileClean), + xactReplyAck -> tileInvalid + )) + } + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, xactInitReadExclusive, xactInitReadShared) + Mux(write || cmd === M_PFW, xactInitReadDirty, xactInitReadClean) } def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, xactInitReadExclusive, outstanding.x_type) + Mux(write, xactInitReadDirty, outstanding.x_type) } - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = Bool(false) - def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { - Mux(outstanding.x_type === xactInitReadExclusive, tileDirty, tileClean) - } - def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = state - def newProbeReply (incoming: ProbeRequest, has_data: Bool): ProbeReply = { - val reply = Wire() { new ProbeReply() } - reply.p_type := probeRepInvalidateAck - reply.global_xact_id := UFix(0) - reply - } - def probeReplyHasData (reply: ProbeReply): Bool = Bool(false) - def transactionInitHasData (init: TransactionInit): Bool = (init.x_type === xactInitWriteUncached) + def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteback //TODO + def getTransactionInitTypeOnWriteback(): Bits = xactInitWriteback + + def hasData (init: TransactionInit): Bool = (init.x_type === xactInitWriteback) + def hasData (reply: TransactionReply) = (reply.x_type === xactReplyData) } trait FourStateCoherence extends CoherencePolicy { diff --git a/uncore/uncore.scala b/uncore/uncore.scala index b907c51d..14164f7f 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -251,7 +251,7 @@ abstract class CoherenceHub(ntiles: Int) extends Component with CoherencePolicy class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence { val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.x_type === xactInitWriteUncached + val is_write = x_init.bits.x_type === xactInitWriteback x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write @@ -260,7 +260,7 @@ class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence io.mem.req_data <> io.tiles(0).xact_init_data val x_rep = io.tiles(0).xact_rep - x_rep.bits.x_type := Mux(io.mem.resp.valid, xactReplyReadExclusive, xactReplyWriteUncached) + x_rep.bits.x_type := Mux(io.mem.resp.valid, xactReplyData, xactReplyAck) x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.bits.data From 5acf1d982014bab2c571faf20317ea92ca8f96c7 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 4 Apr 2012 15:51:33 -0700 Subject: [PATCH 081/374] defined abstract coherence traits in base trait, added Incoherent trait, cleaned up incoherent policy --- uncore/uncore.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index b907c51d..14164f7f 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -251,7 +251,7 @@ abstract class CoherenceHub(ntiles: Int) extends Component with CoherencePolicy class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence { val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.x_type === xactInitWriteUncached + val is_write = x_init.bits.x_type === xactInitWriteback x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write @@ -260,7 +260,7 @@ class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence io.mem.req_data <> io.tiles(0).xact_init_data val x_rep = io.tiles(0).xact_rep - x_rep.bits.x_type := Mux(io.mem.resp.valid, xactReplyReadExclusive, xactReplyWriteUncached) + x_rep.bits.x_type := Mux(io.mem.resp.valid, xactReplyData, xactReplyAck) x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.bits.data From b22d7f81929b8a01e67306e9e5664b9bdd79d46f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 10 Apr 2012 00:09:58 -0700 Subject: [PATCH 082/374] Refactored coherence as member rather than trait. MI and MEI protocols. --- uncore/coherence.scala | 284 +++++++++++++++++++++++++++++++++++++++-- uncore/uncore.scala | 36 +++--- 2 files changed, 289 insertions(+), 31 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 1bd19bbd..e0f5e5c8 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -50,7 +50,7 @@ object cpuCmdToRW { } } -trait CoherencePolicy { +abstract class CoherencePolicy { def isHit (cmd: Bits, state: UFix): Bool def isValid (state: UFix): Bool @@ -72,9 +72,10 @@ trait CoherencePolicy { def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply - def hasData (reply: ProbeReply): Bool - def hasData (init: TransactionInit): Bool - def hasData (reply: TransactionReply): Bool + def messageHasData (reply: ProbeReply): Bool + def messageHasData (init: TransactionInit): Bool + def messageHasData (reply: TransactionReply): Bool + def messageUpdatesDataArray (reply: TransactionReply): Bool def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool def getTransactionReplyType(x_type: UFix, count: UFix): Bits @@ -84,7 +85,14 @@ trait CoherencePolicy { def needsAckReply(x_type: UFix, global_state: UFix): Bool } -trait IncoherentPolicy extends CoherencePolicy { +trait UncachedTransactions { + def getTransactionInitTypeOnUncachedRead(): UFix + def getTransactionInitTypeOnUncachedWrite(): UFix +} + +abstract class CoherencePolicyWithUncached extends CoherencePolicy with UncachedTransactions + +abstract class IncoherentPolicy extends CoherencePolicy { // UNIMPLEMENTED def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = state def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { @@ -93,7 +101,7 @@ trait IncoherentPolicy extends CoherencePolicy { reply.global_xact_id := UFix(0) reply } - def hasData (reply: ProbeReply) = Bool(false) + def messageHasData (reply: ProbeReply) = Bool(false) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = Bool(false) def getTransactionReplyType(x_type: UFix, count: UFix): Bits = Bits(0) def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = UFix(0) @@ -102,7 +110,7 @@ trait IncoherentPolicy extends CoherencePolicy { def needsAckReply(x_type: UFix, global_state: UFix): Bool = Bool(false) } -trait ThreeStateIncoherence extends IncoherentPolicy { +class ThreeStateIncoherence extends IncoherentPolicy { val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } val xactInitReadClean :: xactInitReadDirty :: xactInitWriteback :: Nil = Enum(3){ UFix() } val xactReplyData :: xactReplyAck :: Nil = Enum(2){ UFix() } @@ -141,11 +149,256 @@ trait ThreeStateIncoherence extends IncoherentPolicy { def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteback //TODO def getTransactionInitTypeOnWriteback(): Bits = xactInitWriteback - def hasData (init: TransactionInit): Bool = (init.x_type === xactInitWriteback) - def hasData (reply: TransactionReply) = (reply.x_type === xactReplyData) + def messageHasData (init: TransactionInit): Bool = (init.x_type === xactInitWriteback) + def messageHasData (reply: TransactionReply) = (reply.x_type === xactReplyData) + def messageUpdatesDataArray (reply: TransactionReply) = (reply.x_type === xactReplyData) } -trait FourStateCoherence extends CoherencePolicy { +class TwoStateCoherence extends CoherencePolicyWithUncached { + + val tileInvalid :: tileValid :: Nil = Enum(2){ UFix() } + val globalInvalid :: globalValid :: Nil = Enum(2){ UFix() } + + val xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: Nil = Enum(3){ UFix() } + val xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: Nil = Enum(3){ UFix() } + val probeReqInvalidate :: probeReqCopy :: Nil = Enum(2){ UFix() } + val probeRepInvalidateData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepCopyAck :: Nil = Enum(4){ UFix() } + + def isHit (cmd: Bits, state: UFix): Bool = state != tileInvalid + def isValid (state: UFix): Bool = state != tileInvalid + + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = (outstanding.x_type != xactInitReadExclusive) + def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { + MuxLookup(cmd, (state === tileValid), Array( + M_INV -> (state === tileValid), + M_CLN -> (state === tileValid) + )) + } + def needsWriteback (state: UFix): Bool = { + needsTransactionOnCacheControl(M_INV, state) + } + + def newStateOnHit(cmd: Bits, state: UFix): UFix = state + def newStateOnCacheControl(cmd: Bits) = { + MuxLookup(cmd, tileInvalid, Array( + M_INV -> tileInvalid, + M_CLN -> tileValid + )) + } + def newStateOnWriteback() = newStateOnCacheControl(M_INV) + def newStateOnFlush() = newStateOnCacheControl(M_INV) + def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { + MuxLookup(incoming.x_type, tileInvalid, Array( + xactReplyReadExclusive -> tileValid, + xactReplyReadUncached -> tileInvalid, + xactReplyWriteUncached -> tileInvalid + )) + } + def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { + MuxLookup(incoming.p_type, state, Array( + probeReqInvalidate -> tileInvalid, + probeReqCopy -> state + )) + } + + def getTransactionInitTypeOnUncachedRead() = xactInitReadUncached + def getTransactionInitTypeOnUncachedWrite() = xactInitWriteUncached + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = xactInitReadExclusive + def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = xactInitReadExclusive + def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached + def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) + + def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { + val reply = Wire() { new ProbeReply() } + val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( + probeReqInvalidate -> probeRepInvalidateData, + probeReqCopy -> probeRepCopyData + )) + val without_data = MuxLookup(incoming.p_type, probeRepInvalidateAck, Array( + probeReqInvalidate -> probeRepInvalidateAck, + probeReqCopy -> probeRepCopyAck + )) + reply.p_type := Mux(needsWriteback(state), with_data, without_data) + reply.global_xact_id := incoming.global_xact_id + reply + } + + def messageHasData (reply: ProbeReply): Bool = { + (reply.p_type === probeRepInvalidateData || + reply.p_type === probeRepCopyData) + } + def messageHasData (init: TransactionInit): Bool = { + (init.x_type === xactInitWriteUncached) + } + def messageHasData (reply: TransactionReply): Bool = { + (reply.x_type != xactReplyWriteUncached) + } + def messageUpdatesDataArray (reply: TransactionReply): Bool = { + (reply.x_type === xactReplyReadExclusive) + } + + def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) + + def getTransactionReplyType(x_type: UFix, count: UFix): Bits = { + MuxLookup(x_type, xactReplyReadUncached, Array( + xactInitReadExclusive -> xactReplyReadExclusive, + xactInitReadUncached -> xactReplyReadUncached, + xactInitWriteUncached -> xactReplyWriteUncached + )) + } + + def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = { + MuxLookup(x_type, probeReqCopy, Array( + xactInitReadExclusive -> probeReqInvalidate, + xactInitReadUncached -> probeReqCopy, + xactInitWriteUncached -> probeReqInvalidate + )) + } + + def needsMemRead(x_type: UFix, global_state: UFix): Bool = { + (x_type != xactInitWriteUncached) + } + def needsMemWrite(x_type: UFix, global_state: UFix): Bool = { + (x_type === xactInitWriteUncached) + } + def needsAckReply(x_type: UFix, global_state: UFix): Bool = { + (x_type === xactInitWriteUncached) + } +} + +class ThreeStateCoherence extends CoherencePolicyWithUncached { //MEI + + val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(3){ UFix() } + val globalInvalid :: globalExclusiveClean :: Nil = Enum(2){ UFix() } + + val xactInitReadExclusiveClean :: xactInitReadExclusiveDirty :: xactInitReadUncached :: xactInitWriteUncached :: Nil = Enum(4){ UFix() } + val xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: Nil = Enum(4){ UFix() } + val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() } + val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() } + + def isHit (cmd: Bits, state: UFix): Bool = state != tileInvalid + def isValid (state: UFix): Bool = state != tileInvalid + + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { + val (read, write) = cpuCmdToRW(cmd) + (read && (outstanding.x_type === xactInitReadUncached || outstanding.x_type === xactInitWriteUncached)) || + (write && (outstanding.x_type != xactInitReadExclusiveDirty)) + } + def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { + MuxLookup(cmd, (state === tileExclusiveDirty), Array( + M_INV -> (state === tileExclusiveDirty), + M_CLN -> (state === tileExclusiveDirty) + )) + } + def needsWriteback (state: UFix): Bool = { + needsTransactionOnCacheControl(M_INV, state) + } + + def newStateOnHit(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, tileExclusiveDirty, state) + } + def newStateOnCacheControl(cmd: Bits) = { + MuxLookup(cmd, tileInvalid, Array( + M_INV -> tileInvalid, + M_CLN -> tileExclusiveClean + )) + } + def newStateOnWriteback() = newStateOnCacheControl(M_INV) + def newStateOnFlush() = newStateOnCacheControl(M_INV) + def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { + MuxLookup(incoming.x_type, tileInvalid, Array( + xactReplyReadExclusive -> Mux(outstanding.x_type === xactInitReadExclusiveDirty, tileExclusiveDirty, tileExclusiveClean), + xactReplyReadExclusiveAck -> tileExclusiveDirty, + xactReplyReadUncached -> tileInvalid, + xactReplyWriteUncached -> tileInvalid + )) + } + def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { + MuxLookup(incoming.p_type, state, Array( + probeReqInvalidate -> tileInvalid, + probeReqDowngrade -> tileExclusiveClean, + probeReqCopy -> state + )) + } + + def getTransactionInitTypeOnUncachedRead() = xactInitReadUncached + def getTransactionInitTypeOnUncachedWrite() = xactInitWriteUncached + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, xactInitReadExclusiveDirty, xactInitReadExclusiveClean) + } + def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, xactInitReadExclusiveDirty, outstanding.x_type) + } + def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached + def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) + + def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { + val reply = Wire() { new ProbeReply() } + val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( + probeReqInvalidate -> probeRepInvalidateData, + probeReqDowngrade -> probeRepDowngradeData, + probeReqCopy -> probeRepCopyData + )) + val without_data = MuxLookup(incoming.p_type, probeRepInvalidateAck, Array( + probeReqInvalidate -> probeRepInvalidateAck, + probeReqDowngrade -> probeRepDowngradeAck, + probeReqCopy -> probeRepCopyAck + )) + reply.p_type := Mux(needsWriteback(state), with_data, without_data) + reply.global_xact_id := incoming.global_xact_id + reply + } + + def messageHasData (reply: ProbeReply): Bool = { + (reply.p_type === probeRepInvalidateData || + reply.p_type === probeRepDowngradeData || + reply.p_type === probeRepCopyData) + } + def messageHasData (init: TransactionInit): Bool = { + (init.x_type === xactInitWriteUncached) + } + def messageHasData (reply: TransactionReply): Bool = { + (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck) + } + def messageUpdatesDataArray (reply: TransactionReply): Bool = { + (reply.x_type === xactReplyReadExclusive) + } + + def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) + + def getTransactionReplyType(x_type: UFix, count: UFix): Bits = { + MuxLookup(x_type, xactReplyReadUncached, Array( + xactInitReadExclusiveClean -> xactReplyReadExclusive, + xactInitReadExclusiveDirty -> xactReplyReadExclusive, + xactInitReadUncached -> xactReplyReadUncached, + xactInitWriteUncached -> xactReplyWriteUncached + )) + } + + def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = { + MuxLookup(x_type, probeReqCopy, Array( + xactInitReadExclusiveClean -> probeReqInvalidate, + xactInitReadExclusiveDirty -> probeReqInvalidate, + xactInitReadUncached -> probeReqCopy, + xactInitWriteUncached -> probeReqInvalidate + )) + } + + def needsMemRead(x_type: UFix, global_state: UFix): Bool = { + (x_type != xactInitWriteUncached) + } + def needsMemWrite(x_type: UFix, global_state: UFix): Bool = { + (x_type === xactInitWriteUncached) + } + def needsAckReply(x_type: UFix, global_state: UFix): Bool = { + (x_type === xactInitWriteUncached) + } +} + +class FourStateCoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(4){ UFix() } val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(3){ UFix() } @@ -208,6 +461,8 @@ trait FourStateCoherence extends CoherencePolicy { )) } + def getTransactionInitTypeOnUncachedRead() = xactInitReadUncached + def getTransactionInitTypeOnUncachedWrite() = xactInitWriteUncached def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) Mux(write || cmd === M_PFW, xactInitReadExclusive, xactInitReadShared) @@ -236,17 +491,20 @@ trait FourStateCoherence extends CoherencePolicy { reply } - def hasData (reply: ProbeReply): Bool = { + def messageHasData (reply: ProbeReply): Bool = { (reply.p_type === probeRepInvalidateData || reply.p_type === probeRepDowngradeData || reply.p_type === probeRepCopyData) } - def hasData (init: TransactionInit): Bool = { + def messageHasData (init: TransactionInit): Bool = { (init.x_type === xactInitWriteUncached) } - def hasData (reply: TransactionReply): Bool = { + def messageHasData (reply: TransactionReply): Bool = { (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck) } + def messageUpdatesDataArray (reply: TransactionReply): Bool = { + (reply.x_type === xactReplyReadShared || reply.x_type === xactReplyReadExclusive) + } def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 14164f7f..a8fc9432 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -50,7 +50,7 @@ class ioTileLink extends Bundle { val xact_finish = (new ioDecoupled) { new TransactionFinish } } -class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherence { +class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val io = new Bundle { val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip val p_data = (new ioPipe) { new TrackerProbeData }.flip @@ -140,7 +140,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc io.sharer_count := UFix(ntiles) // TODO: Broadcast only io.x_type := x_type_ - io.mem_req_cmd.valid := Bool(false) + io.mem_req_cmd.valid := Bool(false) io.mem_req_cmd.bits.rw := Bool(false) io.mem_req_cmd.bits.addr := addr_ io.mem_req_cmd.bits.tag := UFix(id) @@ -148,7 +148,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc io.mem_req_data.bits.data := UFix(0) io.mem_req_lock := Bool(false) io.probe_req.valid := Bool(false) - io.probe_req.bits.p_type := getProbeRequestType(x_type_, UFix(0)) + io.probe_req.bits.p_type := co.getProbeRequestType(x_type_, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) io.probe_req.bits.address := addr_ io.push_p_req := Bits(0, width = ntiles) @@ -167,8 +167,8 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc x_type_ := io.alloc_req.bits.xact_init.x_type init_tile_id_ := io.alloc_req.bits.tile_id tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id - x_init_data_needs_write := hasData(io.alloc_req.bits.xact_init) - x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) + x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) + x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) if(ntiles > 1) p_rep_count := UFix(ntiles-1) val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only p_req_flags := p_req_initial_flags @@ -226,7 +226,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { - state := Mux(needsAckReply(x_type_, UFix(0)), s_ack, s_busy) + state := Mux(co.needsAckReply(x_type_, UFix(0)), s_ack, s_busy) } } is(s_ack) { @@ -241,17 +241,17 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc } } -abstract class CoherenceHub(ntiles: Int) extends Component with CoherencePolicy { +abstract class CoherenceHub(ntiles: Int, co: CoherencePolicy) extends Component { val io = new Bundle { val tiles = Vec(ntiles) { new ioTileLink() }.flip val mem = new ioMem } } -class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence +class CoherenceHubNull(co: ThreeStateIncoherence) extends CoherenceHub(1, co) { val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.x_type === xactInitWriteback + val is_write = x_init.bits.x_type === co.xactInitWriteback x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write @@ -260,7 +260,7 @@ class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence io.mem.req_data <> io.tiles(0).xact_init_data val x_rep = io.tiles(0).xact_rep - x_rep.bits.x_type := Mux(io.mem.resp.valid, xactReplyData, xactReplyAck) + x_rep.bits.x_type := Mux(io.mem.resp.valid, co.xactReplyData, co.xactReplyAck) x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.bits.data @@ -275,9 +275,9 @@ class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence } -class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourStateCoherence +class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceHub(ntiles, co) { - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _)) + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _, co)) val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } @@ -344,12 +344,12 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS rep.bits.require_ack := Bool(true) rep.valid := Bool(false) when(io.mem.resp.valid && (UFix(j) === init_tile_id_arr(mem_idx))) { - rep.bits.x_type := getTransactionReplyType(x_type_arr(mem_idx), sh_count_arr(mem_idx)) + rep.bits.x_type := co.getTransactionReplyType(x_type_arr(mem_idx), sh_count_arr(mem_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) rep.bits.global_xact_id := mem_idx rep.valid := Bool(true) } . otherwise { - rep.bits.x_type := getTransactionReplyType(x_type_arr(ack_idx), sh_count_arr(ack_idx)) + rep.bits.x_type := co.getTransactionReplyType(x_type_arr(ack_idx), sh_count_arr(ack_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) rep.bits.global_xact_id := ack_idx when (UFix(j) === init_tile_id_arr(ack_idx)) { @@ -417,16 +417,16 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS val conflicts = Vec(NGLOBAL_XACTS) { Wire() { Bool() } } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(i) := t.busy && x_init.valid && isCoherenceConflict(t.addr, x_init.bits.address) + conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.address) } x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id - want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && hasData(x_init.bits))) + want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits))) x_abort.valid := Bool(false) switch(abort_state_arr(j)) { is(s_idle) { when(want_to_abort_arr(j)) { - when(hasData(x_init.bits)) { + when(co.messageHasData(x_init.bits)) { abort_state_arr(j) := s_abort_drain } . otherwise { abort_state_arr(j) := s_abort_send @@ -478,7 +478,7 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS init_arb.io.in(j).bits.tile_id := UFix(j) val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) val do_pop = foldR(pop_x_inits)(_||_) - x_init_data_dep_list(j).io.enq.valid := do_pop && hasData(x_init.bits) && (abort_state_arr(j) === s_idle) + x_init_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(x_init.bits) && (abort_state_arr(j) === s_idle) x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) x_init.ready := (abort_state_arr(j) === s_abort_complete) || do_pop x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) From 1920c97066f6d1a22f6389621a559243fdfc2ab9 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 10 Apr 2012 00:09:58 -0700 Subject: [PATCH 083/374] Refactored coherence as member rather than trait. MI and MEI protocols. --- uncore/uncore.scala | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 14164f7f..a8fc9432 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -50,7 +50,7 @@ class ioTileLink extends Bundle { val xact_finish = (new ioDecoupled) { new TransactionFinish } } -class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherence { +class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val io = new Bundle { val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip val p_data = (new ioPipe) { new TrackerProbeData }.flip @@ -140,7 +140,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc io.sharer_count := UFix(ntiles) // TODO: Broadcast only io.x_type := x_type_ - io.mem_req_cmd.valid := Bool(false) + io.mem_req_cmd.valid := Bool(false) io.mem_req_cmd.bits.rw := Bool(false) io.mem_req_cmd.bits.addr := addr_ io.mem_req_cmd.bits.tag := UFix(id) @@ -148,7 +148,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc io.mem_req_data.bits.data := UFix(0) io.mem_req_lock := Bool(false) io.probe_req.valid := Bool(false) - io.probe_req.bits.p_type := getProbeRequestType(x_type_, UFix(0)) + io.probe_req.bits.p_type := co.getProbeRequestType(x_type_, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) io.probe_req.bits.address := addr_ io.push_p_req := Bits(0, width = ntiles) @@ -167,8 +167,8 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc x_type_ := io.alloc_req.bits.xact_init.x_type init_tile_id_ := io.alloc_req.bits.tile_id tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id - x_init_data_needs_write := hasData(io.alloc_req.bits.xact_init) - x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) + x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) + x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) if(ntiles > 1) p_rep_count := UFix(ntiles-1) val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only p_req_flags := p_req_initial_flags @@ -226,7 +226,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { - state := Mux(needsAckReply(x_type_, UFix(0)), s_ack, s_busy) + state := Mux(co.needsAckReply(x_type_, UFix(0)), s_ack, s_busy) } } is(s_ack) { @@ -241,17 +241,17 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc } } -abstract class CoherenceHub(ntiles: Int) extends Component with CoherencePolicy { +abstract class CoherenceHub(ntiles: Int, co: CoherencePolicy) extends Component { val io = new Bundle { val tiles = Vec(ntiles) { new ioTileLink() }.flip val mem = new ioMem } } -class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence +class CoherenceHubNull(co: ThreeStateIncoherence) extends CoherenceHub(1, co) { val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.x_type === xactInitWriteback + val is_write = x_init.bits.x_type === co.xactInitWriteback x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write @@ -260,7 +260,7 @@ class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence io.mem.req_data <> io.tiles(0).xact_init_data val x_rep = io.tiles(0).xact_rep - x_rep.bits.x_type := Mux(io.mem.resp.valid, xactReplyData, xactReplyAck) + x_rep.bits.x_type := Mux(io.mem.resp.valid, co.xactReplyData, co.xactReplyAck) x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.bits.data @@ -275,9 +275,9 @@ class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence } -class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourStateCoherence +class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceHub(ntiles, co) { - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _)) + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _, co)) val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } @@ -344,12 +344,12 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS rep.bits.require_ack := Bool(true) rep.valid := Bool(false) when(io.mem.resp.valid && (UFix(j) === init_tile_id_arr(mem_idx))) { - rep.bits.x_type := getTransactionReplyType(x_type_arr(mem_idx), sh_count_arr(mem_idx)) + rep.bits.x_type := co.getTransactionReplyType(x_type_arr(mem_idx), sh_count_arr(mem_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) rep.bits.global_xact_id := mem_idx rep.valid := Bool(true) } . otherwise { - rep.bits.x_type := getTransactionReplyType(x_type_arr(ack_idx), sh_count_arr(ack_idx)) + rep.bits.x_type := co.getTransactionReplyType(x_type_arr(ack_idx), sh_count_arr(ack_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) rep.bits.global_xact_id := ack_idx when (UFix(j) === init_tile_id_arr(ack_idx)) { @@ -417,16 +417,16 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS val conflicts = Vec(NGLOBAL_XACTS) { Wire() { Bool() } } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(i) := t.busy && x_init.valid && isCoherenceConflict(t.addr, x_init.bits.address) + conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.address) } x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id - want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && hasData(x_init.bits))) + want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits))) x_abort.valid := Bool(false) switch(abort_state_arr(j)) { is(s_idle) { when(want_to_abort_arr(j)) { - when(hasData(x_init.bits)) { + when(co.messageHasData(x_init.bits)) { abort_state_arr(j) := s_abort_drain } . otherwise { abort_state_arr(j) := s_abort_send @@ -478,7 +478,7 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS init_arb.io.in(j).bits.tile_id := UFix(j) val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) val do_pop = foldR(pop_x_inits)(_||_) - x_init_data_dep_list(j).io.enq.valid := do_pop && hasData(x_init.bits) && (abort_state_arr(j) === s_idle) + x_init_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(x_init.bits) && (abort_state_arr(j) === s_idle) x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) x_init.ready := (abort_state_arr(j) === s_abort_complete) || do_pop x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) From 2a7d2888a78c4bac3978b5fb79cd6e6513df7e69 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 10 Apr 2012 02:22:45 -0700 Subject: [PATCH 084/374] coherence mostly works now --- uncore/uncore.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index a8fc9432..756c6e7b 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -86,11 +86,11 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { } def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioPipe[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { - req_cmd.valid := !cmd_sent && at_front_of_dep_queue + req_cmd.valid := !cmd_sent && data.valid && at_front_of_dep_queue req_cmd.bits.rw := Bool(true) req_data.valid := data.valid && at_front_of_dep_queue req_data.bits := data.bits - lock := at_front_of_dep_queue + lock := data.valid && at_front_of_dep_queue when(req_cmd.ready && req_cmd.valid) { cmd_sent := Bool(true) } @@ -383,8 +383,8 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) val do_pop = foldR(pop_p_reps)(_ || _) p_rep.ready := Bool(true) - p_rep_data_dep_list(j).io.enq.valid := do_pop - p_rep_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_p_reps) + p_rep_data_dep_list(j).io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits) + p_rep_data_dep_list(j).io.enq.bits.global_xact_id := p_rep.bits.global_xact_id p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) when (p_rep.valid && co.messageHasData(p_rep.bits)) { p_data_valid_arr(idx) := Bool(true) From 98a5d682a59c3a183145ce85628f58e11e8febcf Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 10 Apr 2012 02:22:45 -0700 Subject: [PATCH 085/374] coherence mostly works now --- uncore/uncore.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index a8fc9432..756c6e7b 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -86,11 +86,11 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { } def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioPipe[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { - req_cmd.valid := !cmd_sent && at_front_of_dep_queue + req_cmd.valid := !cmd_sent && data.valid && at_front_of_dep_queue req_cmd.bits.rw := Bool(true) req_data.valid := data.valid && at_front_of_dep_queue req_data.bits := data.bits - lock := at_front_of_dep_queue + lock := data.valid && at_front_of_dep_queue when(req_cmd.ready && req_cmd.valid) { cmd_sent := Bool(true) } @@ -383,8 +383,8 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) val do_pop = foldR(pop_p_reps)(_ || _) p_rep.ready := Bool(true) - p_rep_data_dep_list(j).io.enq.valid := do_pop - p_rep_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_p_reps) + p_rep_data_dep_list(j).io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits) + p_rep_data_dep_list(j).io.enq.bits.global_xact_id := p_rep.bits.global_xact_id p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) when (p_rep.valid && co.messageHasData(p_rep.bits)) { p_data_valid_arr(idx) := Bool(true) From 4a6c7dbc26d9fd2197fb7e21367c0cabb920c28a Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 11 Apr 2012 17:56:59 -0700 Subject: [PATCH 086/374] Policy determined by constants. MSI policy added. --- uncore/coherence.scala | 147 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 143 insertions(+), 4 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index e0f5e5c8..b4b23b4b 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -154,7 +154,7 @@ class ThreeStateIncoherence extends IncoherentPolicy { def messageUpdatesDataArray (reply: TransactionReply) = (reply.x_type === xactReplyData) } -class TwoStateCoherence extends CoherencePolicyWithUncached { +class MICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileValid :: Nil = Enum(2){ UFix() } val globalInvalid :: globalValid :: Nil = Enum(2){ UFix() } @@ -266,7 +266,7 @@ class TwoStateCoherence extends CoherencePolicyWithUncached { } } -class ThreeStateCoherence extends CoherencePolicyWithUncached { //MEI +class MEICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(3){ UFix() } val globalInvalid :: globalExclusiveClean :: Nil = Enum(2){ UFix() } @@ -398,7 +398,146 @@ class ThreeStateCoherence extends CoherencePolicyWithUncached { //MEI } } -class FourStateCoherence extends CoherencePolicyWithUncached { +class MSICoherence extends CoherencePolicyWithUncached { + + val tileInvalid :: tileShared :: tileExclusiveDirty :: Nil = Enum(3){ UFix() } + val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(3){ UFix() } + + val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: Nil = Enum(4){ UFix() } + val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: Nil = Enum(5){ UFix() } + val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() } + val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() } + + def isHit (cmd: Bits, state: UFix): Bool = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, (state === tileExclusiveDirty), + (state === tileShared || state === tileExclusiveDirty)) + } + def isValid (state: UFix): Bool = { + state != tileInvalid + } + + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { + val (read, write) = cpuCmdToRW(cmd) + (read && (outstanding.x_type === xactInitReadUncached || outstanding.x_type === xactInitWriteUncached)) || + (write && (outstanding.x_type != xactInitReadExclusive)) + } + def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { + MuxLookup(cmd, (state === tileExclusiveDirty), Array( + M_INV -> (state === tileExclusiveDirty), + M_CLN -> (state === tileExclusiveDirty) + )) + } + def needsWriteback (state: UFix): Bool = { + needsTransactionOnCacheControl(M_INV, state) + } + + def newStateOnHit(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, tileExclusiveDirty, state) + } + def newStateOnCacheControl(cmd: Bits) = { + MuxLookup(cmd, tileInvalid, Array( + M_INV -> tileInvalid, + M_CLN -> tileShared + )) + } + def newStateOnWriteback() = newStateOnCacheControl(M_INV) + def newStateOnFlush() = newStateOnCacheControl(M_INV) + def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { + MuxLookup(incoming.x_type, tileInvalid, Array( + xactReplyReadShared -> tileShared, + xactReplyReadExclusive -> tileExclusiveDirty, + xactReplyReadExclusiveAck -> tileExclusiveDirty, + xactReplyReadUncached -> tileInvalid, + xactReplyWriteUncached -> tileInvalid + )) + } + def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { + MuxLookup(incoming.p_type, state, Array( + probeReqInvalidate -> tileInvalid, + probeReqDowngrade -> tileShared, + probeReqCopy -> state + )) + } + + def getTransactionInitTypeOnUncachedRead() = xactInitReadUncached + def getTransactionInitTypeOnUncachedWrite() = xactInitWriteUncached + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write || cmd === M_PFW, xactInitReadExclusive, xactInitReadShared) + } + def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, xactInitReadExclusive, outstanding.x_type) + } + def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached + def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) + + def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { + val reply = Wire() { new ProbeReply() } + val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( + probeReqInvalidate -> probeRepInvalidateData, + probeReqDowngrade -> probeRepDowngradeData, + probeReqCopy -> probeRepCopyData + )) + val without_data = MuxLookup(incoming.p_type, probeRepInvalidateAck, Array( + probeReqInvalidate -> probeRepInvalidateAck, + probeReqDowngrade -> probeRepDowngradeAck, + probeReqCopy -> probeRepCopyAck + )) + reply.p_type := Mux(needsWriteback(state), with_data, without_data) + reply.global_xact_id := incoming.global_xact_id + reply + } + + def messageHasData (reply: ProbeReply): Bool = { + (reply.p_type === probeRepInvalidateData || + reply.p_type === probeRepDowngradeData || + reply.p_type === probeRepCopyData) + } + def messageHasData (init: TransactionInit): Bool = { + (init.x_type === xactInitWriteUncached) + } + def messageHasData (reply: TransactionReply): Bool = { + (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck) + } + def messageUpdatesDataArray (reply: TransactionReply): Bool = { + (reply.x_type === xactReplyReadShared || reply.x_type === xactReplyReadExclusive) + } + + def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) + + def getTransactionReplyType(x_type: UFix, count: UFix): Bits = { + MuxLookup(x_type, xactReplyReadUncached, Array( + xactInitReadShared -> Mux(count > UFix(0), xactReplyReadShared, xactReplyReadExclusive), + xactInitReadExclusive -> xactReplyReadExclusive, + xactInitReadUncached -> xactReplyReadUncached, + xactInitWriteUncached -> xactReplyWriteUncached + )) + } + + def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = { + MuxLookup(x_type, probeReqCopy, Array( + xactInitReadShared -> probeReqDowngrade, + xactInitReadExclusive -> probeReqInvalidate, + xactInitReadUncached -> probeReqCopy, + xactInitWriteUncached -> probeReqInvalidate + )) + } + + def needsMemRead(x_type: UFix, global_state: UFix): Bool = { + (x_type != xactInitWriteUncached) + } + def needsMemWrite(x_type: UFix, global_state: UFix): Bool = { + (x_type === xactInitWriteUncached) + } + def needsAckReply(x_type: UFix, global_state: UFix): Bool = { + (x_type === xactInitWriteUncached) + } +} + +class MESICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(4){ UFix() } val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(3){ UFix() } @@ -439,7 +578,7 @@ class FourStateCoherence extends CoherencePolicyWithUncached { def newStateOnCacheControl(cmd: Bits) = { MuxLookup(cmd, tileInvalid, Array( M_INV -> tileInvalid, - M_CLN -> tileExclusiveClean + M_CLN -> tileShared )) } def newStateOnWriteback() = newStateOnCacheControl(M_INV) From 37eb1a4ae615a8e0d8f94fdff342c8bc38204ec1 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 18 Apr 2012 16:31:14 -0700 Subject: [PATCH 087/374] Fixed coherence bug: probe counting for single tile --- uncore/uncore.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 756c6e7b..29d0befd 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -169,14 +169,16 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) - if(ntiles > 1) p_rep_count := UFix(ntiles-1) val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only - p_req_flags := p_req_initial_flags + p_req_flags := p_req_initial_flags(ntiles-1,0) mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id - state := Mux(p_req_initial_flags.orR, s_probe, s_mem) + if(ntiles > 1) { + p_rep_count := UFix(ntiles-1) + state := Mux(p_req_initial_flags(ntiles-1,0).orR, s_probe, s_mem) + } else state := s_mem } } is(s_probe) { From d61e6ee0806466edfc87bc786754290502d4d673 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 18 Apr 2012 16:31:14 -0700 Subject: [PATCH 088/374] Fixed coherence bug: probe counting for single tile --- uncore/uncore.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 756c6e7b..29d0befd 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -169,14 +169,16 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) - if(ntiles > 1) p_rep_count := UFix(ntiles-1) val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only - p_req_flags := p_req_initial_flags + p_req_flags := p_req_initial_flags(ntiles-1,0) mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id - state := Mux(p_req_initial_flags.orR, s_probe, s_mem) + if(ntiles > 1) { + p_rep_count := UFix(ntiles-1) + state := Mux(p_req_initial_flags(ntiles-1,0).orR, s_probe, s_mem) + } else state := s_mem } } is(s_probe) { From 00155f4bc48247b661ffd46831d999d8a35f9918 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 24 Apr 2012 15:11:59 -0700 Subject: [PATCH 089/374] Fixed abort bug: removed uneeded state, added mshr guard on xact_abort.valid and xact_init.ready on same cycle --- uncore/uncore.scala | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 29d0befd..0f482113 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -408,7 +408,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH } // Nack conflicting transaction init attempts - val s_idle :: s_abort_drain :: s_abort_send :: s_abort_complete :: Nil = Enum(4){ UFix() } + val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } val abort_state_arr = Vec(ntiles) { Reg(resetVal = s_idle) } val want_to_abort_arr = Vec(ntiles) { Wire() { Bool()} } for( j <- 0 until ntiles ) { @@ -445,13 +445,10 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH } is(s_abort_send) { // nothing is dequeued for now x_abort.valid := Bool(true) - when(x_abort.ready) { - abort_state_arr(j) := s_abort_complete + when(x_abort.ready) { // raises x_init.ready below + abort_state_arr(j) := s_idle } } - is(s_abort_complete) { // raises x_init.ready below - abort_state_arr(j) := s_idle - } } } @@ -475,6 +472,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_init_data_dep = x_init_data_dep_list(j).io.deq + val x_abort = io.tiles(j).xact_abort init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid init_arb.io.in(j).bits.xact_init := x_init.bits init_arb.io.in(j).bits.tile_id := UFix(j) @@ -482,7 +480,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH val do_pop = foldR(pop_x_inits)(_||_) x_init_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(x_init.bits) && (abort_state_arr(j) === s_idle) x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) - x_init.ready := (abort_state_arr(j) === s_abort_complete) || do_pop + x_init.ready := (x_abort.valid && x_abort.ready) || do_pop x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) x_init_data_dep.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) } From 99bc99f2ad506a8ec5d2e42dbf4447fe19acd9f5 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 24 Apr 2012 15:11:59 -0700 Subject: [PATCH 090/374] Fixed abort bug: removed uneeded state, added mshr guard on xact_abort.valid and xact_init.ready on same cycle --- uncore/uncore.scala | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 29d0befd..0f482113 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -408,7 +408,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH } // Nack conflicting transaction init attempts - val s_idle :: s_abort_drain :: s_abort_send :: s_abort_complete :: Nil = Enum(4){ UFix() } + val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } val abort_state_arr = Vec(ntiles) { Reg(resetVal = s_idle) } val want_to_abort_arr = Vec(ntiles) { Wire() { Bool()} } for( j <- 0 until ntiles ) { @@ -445,13 +445,10 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH } is(s_abort_send) { // nothing is dequeued for now x_abort.valid := Bool(true) - when(x_abort.ready) { - abort_state_arr(j) := s_abort_complete + when(x_abort.ready) { // raises x_init.ready below + abort_state_arr(j) := s_idle } } - is(s_abort_complete) { // raises x_init.ready below - abort_state_arr(j) := s_idle - } } } @@ -475,6 +472,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_init_data_dep = x_init_data_dep_list(j).io.deq + val x_abort = io.tiles(j).xact_abort init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid init_arb.io.in(j).bits.xact_init := x_init.bits init_arb.io.in(j).bits.tile_id := UFix(j) @@ -482,7 +480,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH val do_pop = foldR(pop_x_inits)(_||_) x_init_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(x_init.bits) && (abort_state_arr(j) === s_idle) x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) - x_init.ready := (abort_state_arr(j) === s_abort_complete) || do_pop + x_init.ready := (x_abort.valid && x_abort.ready) || do_pop x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) x_init_data_dep.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) } From f804c57bb02243004a76c6b7d9b9e6dc38454bfb Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 3 May 2012 04:21:11 -0700 Subject: [PATCH 091/374] reduce HTIF clock divider for now --- uncore/slowio.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/uncore/slowio.scala b/uncore/slowio.scala index 395bdc8d..e54996ef 100644 --- a/uncore/slowio.scala +++ b/uncore/slowio.scala @@ -3,7 +3,7 @@ package rocket import Chisel._ import Constants._ -class slowIO[T <: Data](divisor: Int, hold_cycles: Int)(data: => T) extends Component +class slowIO[T <: Data](val divisor: Int, hold_cycles_in: Int = -1)(data: => T) extends Component { val io = new Bundle { val out_fast = new ioDecoupled()(data).flip @@ -15,8 +15,9 @@ class slowIO[T <: Data](divisor: Int, hold_cycles: Int)(data: => T) extends Comp val clk_slow = Bool(OUTPUT) } + val hold_cycles = if (hold_cycles_in == -1) divisor/4 else hold_cycles_in require((divisor & (divisor-1)) == 0) - require(hold_cycles < divisor/2 && hold_cycles >= 2) + require(hold_cycles < divisor/2 && hold_cycles >= 1) val cnt = Reg() { UFix(width = log2up(divisor)) } cnt := cnt + UFix(1) From 0208e9f95e4283d4971fb8b24d1a8cc5f772254d Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Thu, 24 May 2012 10:33:15 -0700 Subject: [PATCH 092/374] removing wires --- uncore/coherence.scala | 10 +++++----- uncore/uncore.scala | 30 +++++++++++++++--------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index b4b23b4b..04843fec 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -96,7 +96,7 @@ abstract class IncoherentPolicy extends CoherencePolicy { // UNIMPLEMENTED def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = state def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - val reply = Wire() { new ProbeReply() } + val reply = new ProbeReply() reply.p_type := UFix(0) reply.global_xact_id := UFix(0) reply @@ -209,7 +209,7 @@ class MICoherence extends CoherencePolicyWithUncached { def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - val reply = Wire() { new ProbeReply() } + val reply = new ProbeReply() val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( probeReqInvalidate -> probeRepInvalidateData, probeReqCopy -> probeRepCopyData @@ -336,7 +336,7 @@ class MEICoherence extends CoherencePolicyWithUncached { def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - val reply = Wire() { new ProbeReply() } + val reply = new ProbeReply() val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( probeReqInvalidate -> probeRepInvalidateData, probeReqDowngrade -> probeRepDowngradeData, @@ -475,7 +475,7 @@ class MSICoherence extends CoherencePolicyWithUncached { def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - val reply = Wire() { new ProbeReply() } + val reply = new ProbeReply() val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( probeReqInvalidate -> probeRepInvalidateData, probeReqDowngrade -> probeRepDowngradeData, @@ -614,7 +614,7 @@ class MESICoherence extends CoherencePolicyWithUncached { def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - val reply = Wire() { new ProbeReply() } + val reply = new ProbeReply() val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( probeReqInvalidate -> probeRepInvalidateData, probeReqDowngrade -> probeRepDowngradeData, diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 0f482113..ab56d0f4 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -281,20 +281,20 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH { val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _, co)) - val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } - val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val x_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_MAX_BITS)} } - val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val busy_arr = Vec(NGLOBAL_XACTS){ Bool() } + val addr_arr = Vec(NGLOBAL_XACTS){ Bits(width=PADDR_BITS-OFFSET_BITS) } + val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_ID_BITS) } + val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_XACT_ID_BITS) } + val x_type_arr = Vec(NGLOBAL_XACTS){ Bits(width=X_INIT_TYPE_MAX_BITS) } + val sh_count_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_ID_BITS) } + val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } - val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } - val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } - val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } - val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bits(width = TILE_ID_BITS)} } - val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } + val do_free_arr = Vec(NGLOBAL_XACTS){ Bool() } + val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Bool()} } + val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Bool()} } + val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } + val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width = TILE_ID_BITS) } + val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io @@ -410,13 +410,13 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH // Nack conflicting transaction init attempts val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } val abort_state_arr = Vec(ntiles) { Reg(resetVal = s_idle) } - val want_to_abort_arr = Vec(ntiles) { Wire() { Bool()} } + val want_to_abort_arr = Vec(ntiles) { Bool() } for( j <- 0 until ntiles ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_abort = io.tiles(j).xact_abort val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) - val conflicts = Vec(NGLOBAL_XACTS) { Wire() { Bool() } } + val conflicts = Vec(NGLOBAL_XACTS) { Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.address) From 6f2f1ba21c573392a7b21f5afa9c863bb9114abe Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Thu, 24 May 2012 10:33:15 -0700 Subject: [PATCH 093/374] removing wires --- uncore/uncore.scala | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 0f482113..ab56d0f4 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -281,20 +281,20 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH { val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _, co)) - val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } - val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val x_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_MAX_BITS)} } - val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val busy_arr = Vec(NGLOBAL_XACTS){ Bool() } + val addr_arr = Vec(NGLOBAL_XACTS){ Bits(width=PADDR_BITS-OFFSET_BITS) } + val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_ID_BITS) } + val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_XACT_ID_BITS) } + val x_type_arr = Vec(NGLOBAL_XACTS){ Bits(width=X_INIT_TYPE_MAX_BITS) } + val sh_count_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_ID_BITS) } + val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } - val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } - val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } - val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } - val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bits(width = TILE_ID_BITS)} } - val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } + val do_free_arr = Vec(NGLOBAL_XACTS){ Bool() } + val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Bool()} } + val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Bool()} } + val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } + val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width = TILE_ID_BITS) } + val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io @@ -410,13 +410,13 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH // Nack conflicting transaction init attempts val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } val abort_state_arr = Vec(ntiles) { Reg(resetVal = s_idle) } - val want_to_abort_arr = Vec(ntiles) { Wire() { Bool()} } + val want_to_abort_arr = Vec(ntiles) { Bool() } for( j <- 0 until ntiles ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_abort = io.tiles(j).xact_abort val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) - val conflicts = Vec(NGLOBAL_XACTS) { Wire() { Bool() } } + val conflicts = Vec(NGLOBAL_XACTS) { Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.address) From f2942f79f98ad17c3fbd423d0896b3a7631b3d21 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Wed, 6 Jun 2012 12:47:17 -0700 Subject: [PATCH 094/374] moving util out into Chisel standard library --- uncore/uncore.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index ab56d0f4..9791ea0f 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -120,7 +120,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val x_type_ = Reg{ Bits() } val init_tile_id_ = Reg{ Bits() } val tile_xact_id_ = Reg{ Bits() } - val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2up(ntiles))) + val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(ntiles))) val p_req_flags = Reg(resetVal = Bits(0, width = ntiles)) val p_rep_tile_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) @@ -128,9 +128,9 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val p_rep_data_needs_write = Reg(resetVal = Bool(false)) val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val mem_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) - val mem_cnt_max = ~UFix(0, width = log2up(REFILL_CYCLES)) + val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) io.busy := state != s_idle io.addr := addr_ @@ -415,7 +415,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_abort = io.tiles(j).xact_abort - val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val conflicts = Vec(NGLOBAL_XACTS) { Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io @@ -438,7 +438,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH is(s_abort_drain) { // raises x_init_data.ready below when(x_init_data.valid) { abort_cnt := abort_cnt + UFix(1) - when(abort_cnt === ~UFix(0, width = log2up(REFILL_CYCLES))) { + when(abort_cnt === ~UFix(0, width = log2Up(REFILL_CYCLES))) { abort_state_arr(j) := s_abort_send } } From 9b3161920f9a2f0b0905f2df4ee9788a413a3d6e Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Wed, 6 Jun 2012 12:47:17 -0700 Subject: [PATCH 095/374] moving util out into Chisel standard library --- uncore/memserdes.scala | 12 ++++++------ uncore/slowio.scala | 4 ++-- uncore/uncore.scala | 10 +++++----- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/uncore/memserdes.scala b/uncore/memserdes.scala index 27bbaa95..8dc6a8ae 100644 --- a/uncore/memserdes.scala +++ b/uncore/memserdes.scala @@ -26,8 +26,8 @@ class MemSerdes extends Component val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(5) { UFix() } val state = Reg(resetVal = s_idle) - val send_cnt = Reg(resetVal = UFix(0, log2up((max(abits, dbits)+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) - val data_send_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) + val send_cnt = Reg(resetVal = UFix(0, log2Up((max(abits, dbits)+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) + val data_send_cnt = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) val adone = io.narrow.req.ready && send_cnt === UFix((abits-1)/MEM_BACKUP_WIDTH) val ddone = io.narrow.req.ready && send_cnt === UFix((dbits-1)/MEM_BACKUP_WIDTH) @@ -67,8 +67,8 @@ class MemSerdes extends Component send_cnt := UFix(0) } - val recv_cnt = Reg(resetVal = UFix(0, log2up((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) - val data_recv_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) + val recv_cnt = Reg(resetVal = UFix(0, log2Up((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) + val data_recv_cnt = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) val resp_val = Reg(resetVal = Bool(false)) resp_val := Bool(false) @@ -98,8 +98,8 @@ class MemDessert extends Component // test rig side val rbits = io.wide.resp.bits.getWidth require(dbits >= abits && rbits >= dbits) - val recv_cnt = Reg(resetVal = UFix(0, log2up((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) - val data_recv_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) + val recv_cnt = Reg(resetVal = UFix(0, log2Up((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) + val data_recv_cnt = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) val adone = io.narrow.req.valid && recv_cnt === UFix((abits-1)/MEM_BACKUP_WIDTH) val ddone = io.narrow.req.valid && recv_cnt === UFix((dbits-1)/MEM_BACKUP_WIDTH) val rdone = io.narrow.resp.valid && recv_cnt === UFix((rbits-1)/MEM_BACKUP_WIDTH) diff --git a/uncore/slowio.scala b/uncore/slowio.scala index e54996ef..0c513849 100644 --- a/uncore/slowio.scala +++ b/uncore/slowio.scala @@ -19,7 +19,7 @@ class slowIO[T <: Data](val divisor: Int, hold_cycles_in: Int = -1)(data: => T) require((divisor & (divisor-1)) == 0) require(hold_cycles < divisor/2 && hold_cycles >= 1) - val cnt = Reg() { UFix(width = log2up(divisor)) } + val cnt = Reg() { UFix(width = log2Up(divisor)) } cnt := cnt + UFix(1) val out_en = cnt === UFix(divisor/2+hold_cycles-1) // rising edge + hold time val in_en = cnt === UFix(divisor/2-1) // rising edge @@ -46,5 +46,5 @@ class slowIO[T <: Data](val divisor: Int, hold_cycles_in: Int = -1)(data: => T) io.in_slow.ready := in_slow_rdy io.out_slow.valid := out_slow_val io.out_slow.bits := out_slow_bits - io.clk_slow := cnt(log2up(divisor)-1).toBool + io.clk_slow := cnt(log2Up(divisor)-1).toBool } diff --git a/uncore/uncore.scala b/uncore/uncore.scala index ab56d0f4..9791ea0f 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -120,7 +120,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val x_type_ = Reg{ Bits() } val init_tile_id_ = Reg{ Bits() } val tile_xact_id_ = Reg{ Bits() } - val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2up(ntiles))) + val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(ntiles))) val p_req_flags = Reg(resetVal = Bits(0, width = ntiles)) val p_rep_tile_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) @@ -128,9 +128,9 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val p_rep_data_needs_write = Reg(resetVal = Bool(false)) val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val mem_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) - val mem_cnt_max = ~UFix(0, width = log2up(REFILL_CYCLES)) + val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) io.busy := state != s_idle io.addr := addr_ @@ -415,7 +415,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_abort = io.tiles(j).xact_abort - val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val conflicts = Vec(NGLOBAL_XACTS) { Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io @@ -438,7 +438,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH is(s_abort_drain) { // raises x_init_data.ready below when(x_init_data.valid) { abort_cnt := abort_cnt + UFix(1) - when(abort_cnt === ~UFix(0, width = log2up(REFILL_CYCLES))) { + when(abort_cnt === ~UFix(0, width = log2Up(REFILL_CYCLES))) { abort_state_arr(j) := s_abort_send } } From 0c6bade592ca5c6216413831d5464b5850e6ec58 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Wed, 6 Jun 2012 18:22:56 -0700 Subject: [PATCH 096/374] ioDecoupled -> FIFOIO, ioPipe -> PipeIO --- uncore/uncore.scala | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 9791ea0f..6239981f 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -21,9 +21,9 @@ class MemResp () extends MemData class ioMem() extends Bundle { - val req_cmd = (new ioDecoupled) { new MemReqCmd() } - val req_data = (new ioDecoupled) { new MemData() } - val resp = (new ioPipe) { new MemResp() }.flip + val req_cmd = (new FIFOIO) { new MemReqCmd() } + val req_data = (new FIFOIO) { new MemData() } + val resp = (new PipeIO) { new MemResp() }.flip } class TrackerProbeData extends Bundle { @@ -40,34 +40,34 @@ class TrackerDependency extends Bundle { } class ioTileLink extends Bundle { - val xact_init = (new ioDecoupled) { new TransactionInit } - val xact_init_data = (new ioDecoupled) { new TransactionInitData } - val xact_abort = (new ioDecoupled) { new TransactionAbort }.flip - val probe_req = (new ioDecoupled) { new ProbeRequest }.flip - val probe_rep = (new ioDecoupled) { new ProbeReply } - val probe_rep_data = (new ioDecoupled) { new ProbeReplyData } - val xact_rep = (new ioPipe) { new TransactionReply }.flip - val xact_finish = (new ioDecoupled) { new TransactionFinish } + val xact_init = (new FIFOIO) { new TransactionInit } + val xact_init_data = (new FIFOIO) { new TransactionInitData } + val xact_abort = (new FIFOIO) { new TransactionAbort }.flip + val probe_req = (new FIFOIO) { new ProbeRequest }.flip + val probe_rep = (new FIFOIO) { new ProbeReply } + val probe_rep_data = (new FIFOIO) { new ProbeReplyData } + val xact_rep = (new PipeIO) { new TransactionReply }.flip + val xact_finish = (new FIFOIO) { new TransactionFinish } } class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val io = new Bundle { - val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip - val p_data = (new ioPipe) { new TrackerProbeData }.flip + val alloc_req = (new FIFOIO) { new TrackerAllocReq }.flip + val p_data = (new PipeIO) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) val p_rep_cnt_dec = Bits(ntiles, INPUT) val p_req_cnt_inc = Bits(ntiles, INPUT) - val p_rep_data = (new ioPipe) { new ProbeReplyData }.flip - val x_init_data = (new ioPipe) { new TransactionInitData }.flip + val p_rep_data = (new PipeIO) { new ProbeReplyData }.flip + val x_init_data = (new PipeIO) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) - val p_rep_data_dep = (new ioPipe) { new TrackerDependency }.flip - val x_init_data_dep = (new ioPipe) { new TrackerDependency }.flip + val p_rep_data_dep = (new PipeIO) { new TrackerDependency }.flip + val x_init_data_dep = (new PipeIO) { new TrackerDependency }.flip - val mem_req_cmd = (new ioDecoupled) { new MemReqCmd } - val mem_req_data = (new ioDecoupled) { new MemData } + val mem_req_cmd = (new FIFOIO) { new MemReqCmd } + val mem_req_data = (new FIFOIO) { new MemData } val mem_req_lock = Bool(OUTPUT) - val probe_req = (new ioDecoupled) { new ProbeRequest } + val probe_req = (new FIFOIO) { new ProbeRequest } val busy = Bool(OUTPUT) val addr = Bits(PADDR_BITS - OFFSET_BITS, OUTPUT) val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) @@ -85,7 +85,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val send_x_rep_ack = Bool(OUTPUT) } - def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioPipe[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { + def doMemReqWrite(req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: PipeIO[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { req_cmd.valid := !cmd_sent && data.valid && at_front_of_dep_queue req_cmd.bits.rw := Bool(true) req_data.valid := data.valid && at_front_of_dep_queue @@ -106,7 +106,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { } } - def doMemReqRead(req_cmd: ioDecoupled[MemReqCmd], trigger: Bool) { + def doMemReqRead(req_cmd: FIFOIO[MemReqCmd], trigger: Bool) { req_cmd.valid := Bool(true) req_cmd.bits.rw := Bool(false) when(req_cmd.ready) { From 166b85705533ef7d6d37ea049617b211087f7b1e Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Wed, 6 Jun 2012 18:22:56 -0700 Subject: [PATCH 097/374] ioDecoupled -> FIFOIO, ioPipe -> PipeIO --- uncore/memserdes.scala | 4 ++-- uncore/slowio.scala | 8 ++++---- uncore/uncore.scala | 44 +++++++++++++++++++++--------------------- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/uncore/memserdes.scala b/uncore/memserdes.scala index 8dc6a8ae..e20f1ec3 100644 --- a/uncore/memserdes.scala +++ b/uncore/memserdes.scala @@ -7,8 +7,8 @@ import scala.math._ class ioMemSerialized extends Bundle { - val req = (new ioDecoupled) { Bits(width = MEM_BACKUP_WIDTH) } - val resp = (new ioPipe) { Bits(width = MEM_BACKUP_WIDTH) }.flip + val req = (new FIFOIO) { Bits(width = MEM_BACKUP_WIDTH) } + val resp = (new PipeIO) { Bits(width = MEM_BACKUP_WIDTH) }.flip } class MemSerdes extends Component diff --git a/uncore/slowio.scala b/uncore/slowio.scala index 0c513849..6cf5a3d9 100644 --- a/uncore/slowio.scala +++ b/uncore/slowio.scala @@ -6,11 +6,11 @@ import Constants._ class slowIO[T <: Data](val divisor: Int, hold_cycles_in: Int = -1)(data: => T) extends Component { val io = new Bundle { - val out_fast = new ioDecoupled()(data).flip - val out_slow = new ioDecoupled()(data) + val out_fast = new FIFOIO()(data).flip + val out_slow = new FIFOIO()(data) - val in_fast = new ioDecoupled()(data) - val in_slow = new ioDecoupled()(data).flip + val in_fast = new FIFOIO()(data) + val in_slow = new FIFOIO()(data).flip val clk_slow = Bool(OUTPUT) } diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 9791ea0f..6239981f 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -21,9 +21,9 @@ class MemResp () extends MemData class ioMem() extends Bundle { - val req_cmd = (new ioDecoupled) { new MemReqCmd() } - val req_data = (new ioDecoupled) { new MemData() } - val resp = (new ioPipe) { new MemResp() }.flip + val req_cmd = (new FIFOIO) { new MemReqCmd() } + val req_data = (new FIFOIO) { new MemData() } + val resp = (new PipeIO) { new MemResp() }.flip } class TrackerProbeData extends Bundle { @@ -40,34 +40,34 @@ class TrackerDependency extends Bundle { } class ioTileLink extends Bundle { - val xact_init = (new ioDecoupled) { new TransactionInit } - val xact_init_data = (new ioDecoupled) { new TransactionInitData } - val xact_abort = (new ioDecoupled) { new TransactionAbort }.flip - val probe_req = (new ioDecoupled) { new ProbeRequest }.flip - val probe_rep = (new ioDecoupled) { new ProbeReply } - val probe_rep_data = (new ioDecoupled) { new ProbeReplyData } - val xact_rep = (new ioPipe) { new TransactionReply }.flip - val xact_finish = (new ioDecoupled) { new TransactionFinish } + val xact_init = (new FIFOIO) { new TransactionInit } + val xact_init_data = (new FIFOIO) { new TransactionInitData } + val xact_abort = (new FIFOIO) { new TransactionAbort }.flip + val probe_req = (new FIFOIO) { new ProbeRequest }.flip + val probe_rep = (new FIFOIO) { new ProbeReply } + val probe_rep_data = (new FIFOIO) { new ProbeReplyData } + val xact_rep = (new PipeIO) { new TransactionReply }.flip + val xact_finish = (new FIFOIO) { new TransactionFinish } } class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val io = new Bundle { - val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip - val p_data = (new ioPipe) { new TrackerProbeData }.flip + val alloc_req = (new FIFOIO) { new TrackerAllocReq }.flip + val p_data = (new PipeIO) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) val p_rep_cnt_dec = Bits(ntiles, INPUT) val p_req_cnt_inc = Bits(ntiles, INPUT) - val p_rep_data = (new ioPipe) { new ProbeReplyData }.flip - val x_init_data = (new ioPipe) { new TransactionInitData }.flip + val p_rep_data = (new PipeIO) { new ProbeReplyData }.flip + val x_init_data = (new PipeIO) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) - val p_rep_data_dep = (new ioPipe) { new TrackerDependency }.flip - val x_init_data_dep = (new ioPipe) { new TrackerDependency }.flip + val p_rep_data_dep = (new PipeIO) { new TrackerDependency }.flip + val x_init_data_dep = (new PipeIO) { new TrackerDependency }.flip - val mem_req_cmd = (new ioDecoupled) { new MemReqCmd } - val mem_req_data = (new ioDecoupled) { new MemData } + val mem_req_cmd = (new FIFOIO) { new MemReqCmd } + val mem_req_data = (new FIFOIO) { new MemData } val mem_req_lock = Bool(OUTPUT) - val probe_req = (new ioDecoupled) { new ProbeRequest } + val probe_req = (new FIFOIO) { new ProbeRequest } val busy = Bool(OUTPUT) val addr = Bits(PADDR_BITS - OFFSET_BITS, OUTPUT) val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) @@ -85,7 +85,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val send_x_rep_ack = Bool(OUTPUT) } - def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioPipe[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { + def doMemReqWrite(req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: PipeIO[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { req_cmd.valid := !cmd_sent && data.valid && at_front_of_dep_queue req_cmd.bits.rw := Bool(true) req_data.valid := data.valid && at_front_of_dep_queue @@ -106,7 +106,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { } } - def doMemReqRead(req_cmd: ioDecoupled[MemReqCmd], trigger: Bool) { + def doMemReqRead(req_cmd: FIFOIO[MemReqCmd], trigger: Bool) { req_cmd.valid := Bool(true) req_cmd.bits.rw := Bool(false) when(req_cmd.ready) { From 66cf6902619c60f9b181c52653b14e1e70b4375a Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 10 Jul 2012 05:23:29 -0700 Subject: [PATCH 098/374] add L2$ It still has performance bugs but no correctness bugs AFAIK. --- uncore/llc.scala | 392 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 392 insertions(+) create mode 100644 uncore/llc.scala diff --git a/uncore/llc.scala b/uncore/llc.scala new file mode 100644 index 00000000..71dfa536 --- /dev/null +++ b/uncore/llc.scala @@ -0,0 +1,392 @@ +package rocket + +import Chisel._ +import Node._ +import Constants._ + +class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component +{ + val io = new Bundle { + val addr = UFix(log2Up(n), INPUT) + val en = Bool(INPUT) + val rw = Bool(INPUT) + val wdata = gen.asInput + val wmask = gen.asInput + val rdata = gen.asOutput + } + require(readLatency >= 0 && readLatency <= 2) + val data = gen + val colMux = if (2*data.width <= leaf.data.width && n > leaf.n) 1 << math.floor(math.log(leaf.data.width/data.width)/math.log(2)).toInt else 1 + val nWide = if (data.width > leaf.data.width) 1+(data.width-1)/leaf.data.width else 1 + val nDeep = if (n > colMux*leaf.n) 1+(n-1)/(colMux*leaf.n) else 1 + if (nDeep > 1 || colMux > 1) + require(isPow2(n) && isPow2(leaf.n)) + + val idx = io.addr(log2Up(n/nDeep/colMux)-1, 0) + val rdataDeep = Vec(nDeep) { Bits() } + val rdataSel = Vec(nDeep) { Bool() } + val cond = Vec(nDeep) { Bool() } + val ren = Vec(nDeep) { Bool() } + val reg_ren = Vec(nDeep) { Reg() { Bool() } } + val reg2_ren = Vec(nDeep) { Reg() { Bool() } } + val reg_raddr = Vec(nDeep) { Reg() { UFix() } } + val reg2_raddr = Vec(nDeep) { Reg() { UFix() } } + val renOut = Vec(nDeep) { Bool() } + val raddrOut = Vec(nDeep) { UFix() } + val rdata = Vec(nDeep) { Vec(nWide) { Bits() } } + val wdata = io.wdata.toBits + val wmask = io.wmask.toBits + for (i <- 0 until nDeep) { + cond(i) := (if (nDeep == 1) io.en else io.en && UFix(i) === io.addr(log2Up(n)-1, log2Up(n/nDeep))) + ren(i) := cond(i) && !io.rw + reg_ren(i) := ren(i) + reg2_ren(i) := reg_ren(i) + when (ren(i)) { reg_raddr(i) := io.addr } + when (reg_ren(i)) { reg2_raddr(i) := reg_raddr(i) } + renOut(i) := (if (readLatency > 1) reg2_ren(i) else if (readLatency > 0) reg_ren(i) else ren(i)) + raddrOut(i) := (if (readLatency > 1) reg2_raddr(i) else if (readLatency > 0) reg_raddr(i) else io.addr) + + for (j <- 0 until nWide) { + val mem = leaf.clone + var dout: Bits = null + val dout1 = if (readLatency > 0) Reg() { Bits() } else null + + var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) + if (colMux > 1) + wmask0 = wmask0 & FillInterleaved(gen.width, UFixToOH(io.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) + val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) + when (cond(i)) { + when (io.rw) { mem.write(idx, wdata0, wmask0) } + .otherwise { if (readLatency > 0) dout1 := mem(idx) } + } + + if (readLatency == 0) { + dout = mem(idx) + } else if (readLatency == 1) { + dout = dout1 + } else { + val dout2 = Reg() { Bits() } + when (reg_ren(i)) { dout2 := dout1 } + dout = dout2 + } + + rdata(i)(j) := dout + } + val rdataWide = rdata(i).reduceLeft((x, y) => Cat(y, x)) + + var colMuxOut = rdataWide + if (colMux > 1) { + val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.width*(k+1)-1, gen.width*k))) { Bits() } + colMuxOut = colMuxIn(raddrOut(i)(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) + } + + rdataDeep(i) := colMuxOut + rdataSel(i) := renOut(i) + } + + io.rdata := Mux1H(rdataSel, rdataDeep) +} + +class LLCDataReq(ways: Int) extends MemReqCmd +{ + val way = UFix(width = log2Up(ways)) + val isWriteback = Bool() + + override def clone = new LLCDataReq(ways).asInstanceOf[this.type] +} + +class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component +{ + val io = new Bundle { + val cpu = (new FIFOIO) { new MemReqCmd }.flip + val repl_way = UFix(log2Up(ways), INPUT) + val repl_dirty = Bool(INPUT) + val repl_tag = UFix(PADDR_BITS - OFFSET_BITS - log2Up(sets), INPUT) + val data = (new FIFOIO) { new LLCDataReq(ways) } + val tag = (new PipeIO) { new Bundle { + val addr = UFix(width = PADDR_BITS - OFFSET_BITS) + val way = UFix(width = log2Up(ways)) + } } + val mem = new ioMem + val mem_resp_set = UFix(log2Up(sets), OUTPUT) + val mem_resp_way = UFix(log2Up(ways), OUTPUT) + } + + class MSHR extends Bundle { + val addr = UFix(width = PADDR_BITS - OFFSET_BITS) + val way = UFix(width = log2Up(ways)) + val tag = io.cpu.bits.tag.clone + val refilled = Bool() + val refillCount = UFix(width = log2Up(REFILL_CYCLES)) + val requested = Bool() + val old_dirty = Bool() + val old_tag = UFix(width = PADDR_BITS - OFFSET_BITS - log2Up(sets)) + + override def clone = new MSHR().asInstanceOf[this.type] + } + + val valid = Vec(outstanding) { Reg(resetVal = Bool(false)) } + val validBits = valid.toBits + val freeId = PriorityEncoder(~validBits) + val mshr = Vec(outstanding) { Reg() { new MSHR } } + when (io.cpu.valid && io.cpu.ready) { + valid(freeId) := Bool(true) + mshr(freeId).addr := io.cpu.bits.addr + mshr(freeId).tag := io.cpu.bits.tag + mshr(freeId).way := io.repl_way + mshr(freeId).old_dirty := io.repl_dirty + mshr(freeId).old_tag := io.repl_tag + mshr(freeId).requested := Bool(false) + mshr(freeId).refillCount := UFix(0) + mshr(freeId).refilled := Bool(false) + } + + val requests = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && !mshr(i).old_dirty && !mshr(i).requested):_*) + val request = requests.orR + val requestId = PriorityEncoder(requests) + when (io.mem.req_cmd.valid && io.mem.req_cmd.ready) { mshr(requestId).requested := Bool(true) } + + val refillId = io.mem.resp.bits.tag(log2Up(outstanding)-1, 0) + val refillCount = mshr(refillId).refillCount + when (io.mem.resp.valid) { + mshr(refillId).refillCount := refillCount + UFix(1) + when (refillCount === UFix(REFILL_CYCLES-1)) { mshr(refillId).refilled := Bool(true) } + } + + val replays = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).refilled):_*) + val replay = replays.orR + val replayId = PriorityEncoder(replays) + when (replay && io.data.ready) { valid(replayId) := Bool(false) } + + val writebacks = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).old_dirty):_*) + val writeback = writebacks.orR + val writebackId = PriorityEncoder(writebacks) + when (writeback && io.data.ready && !replay) { mshr(writebackId).old_dirty := Bool(false) } + + val conflicts = Cat(Bits(0), (0 until outstanding).map(i => valid(i) && io.cpu.bits.addr(log2Up(sets)-1, 0) === mshr(i).addr(log2Up(sets)-1, 0)):_*) + io.cpu.ready := !conflicts.orR && !validBits.andR + + io.data.valid := replay || writeback + io.data.bits.rw := Bool(false) + io.data.bits.tag := mshr(replayId).tag + io.data.bits.isWriteback := Bool(true) + io.data.bits.addr := Cat(mshr(writebackId).old_tag, mshr(writebackId).addr(log2Up(sets)-1, 0)).toUFix + io.data.bits.way := mshr(writebackId).way + when (replay) { + io.data.bits.isWriteback := Bool(false) + io.data.bits.addr := mshr(replayId).addr + io.data.bits.way := mshr(replayId).way + } + io.tag.valid := replay && io.data.ready + io.tag.bits.addr := io.data.bits.addr + io.tag.bits.way := io.data.bits.way + + io.mem.req_cmd.valid := request + io.mem.req_cmd.bits.rw := Bool(false) + io.mem.req_cmd.bits.addr := mshr(requestId).addr + io.mem.req_cmd.bits.tag := requestId + io.mem_resp_set := mshr(refillId).addr + io.mem_resp_way := mshr(refillId).way +} + +class LLCWriteback(requestors: Int) extends Component +{ + val io = new Bundle { + val req = Vec(requestors) { (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) }.flip } + val data = Vec(requestors) { (new FIFOIO) { new MemData }.flip } + val mem = new ioMem + } + + val valid = Reg(resetVal = Bool(false)) + val who = Reg() { UFix() } + val addr = Reg() { UFix() } + val cmd_sent = Reg() { Bool() } + val data_sent = Reg() { Bool() } + val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) + + var anyReq = Bool(false) + for (i <- 0 until requestors) { + io.req(i).ready := !valid && !anyReq + io.data(i).ready := valid && who === UFix(i) && io.mem.req_data.ready + anyReq = anyReq || io.req(i).valid + } + + val nextWho = PriorityEncoder(io.req.map(_.valid)) + when (!valid && io.req.map(_.valid).reduceLeft(_||_)) { + valid := Bool(true) + cmd_sent := Bool(false) + data_sent := Bool(false) + who := nextWho + addr := io.req(nextWho).bits + } + + when (io.mem.req_data.valid && io.mem.req_data.ready) { + count := count + UFix(1) + when (count === UFix(REFILL_CYCLES-1)) { + data_sent := Bool(true) + when (cmd_sent) { valid := Bool(false) } + } + } + when (io.mem.req_cmd.valid && io.mem.req_cmd.ready) { cmd_sent := Bool(true) } + when (valid && cmd_sent && data_sent) { valid := Bool(false) } + + io.mem.req_cmd.valid := valid && !cmd_sent + io.mem.req_cmd.bits.addr := addr + io.mem.req_cmd.bits.rw := Bool(true) + + io.mem.req_data.valid := valid && !data_sent && io.data(who).valid + io.mem.req_data.bits := io.data(who).bits +} + +class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component +{ + val io = new Bundle { + val req = (new FIFOIO) { new LLCDataReq(ways) }.flip + val req_data = (new FIFOIO) { new MemData }.flip + val writeback = (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) } + val writeback_data = (new FIFOIO) { new MemData } + val resp = (new PipeIO) { new MemResp } + val mem_resp = (new PipeIO) { new MemResp }.flip + val mem_resp_set = UFix(log2Up(sets), INPUT) + val mem_resp_way = UFix(log2Up(ways), INPUT) + } + + val data = new BigMem(sets*ways*REFILL_CYCLES, 2, leaf)(Bits(width = MEM_DATA_BITS)) + class QEntry extends MemResp { + val isWriteback = Bool() + override def clone = new QEntry().asInstanceOf[this.type] + } + val q = (new queue(4)) { new QEntry } + val qReady = q.io.count <= UFix(q.entries - 3) + val valid = Reg(resetVal = Bool(false)) + val req = Reg() { io.req.bits.clone } + val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) + val refillCount = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) + + when (data.io.en && !io.mem_resp.valid) { + count := count + UFix(1) + when (valid && count === UFix(REFILL_CYCLES-1)) { valid := Bool(false) } + } + when (io.req.valid && io.req.ready) { valid := Bool(true); req := io.req.bits } + when (io.mem_resp.valid) { refillCount := refillCount + UFix(1) } + + data.io.en := io.req.valid && io.req.ready && Mux(io.req.bits.rw, io.req_data.valid, qReady) + data.io.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUFix + data.io.rw := io.req.bits.rw + data.io.wdata := io.req_data.bits.data + data.io.wmask := Fix(-1, io.req_data.bits.data.width) + when (valid) { + data.io.en := Mux(req.rw, io.req_data.valid, qReady) + data.io.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUFix + data.io.rw := req.rw + } + when (io.mem_resp.valid) { + data.io.en := Bool(true) + data.io.addr := Cat(io.mem_resp_way, io.mem_resp_set, refillCount).toUFix + data.io.rw := Bool(true) + data.io.wdata := io.mem_resp.bits.data + } + + q.io.enq.valid := Reg(Reg(data.io.en && !data.io.rw, resetVal = Bool(false)), resetVal = Bool(false)) + q.io.enq.bits.tag := Reg(Reg(Mux(valid, req.tag, io.req.bits.tag))) + q.io.enq.bits.data := data.io.rdata + q.io.enq.bits.isWriteback := Reg(Reg(Mux(valid, req.isWriteback, io.req.bits.isWriteback))) + + io.req.ready := !valid && Mux(io.req.bits.isWriteback, io.writeback.ready, Bool(true)) + io.req_data.ready := !io.mem_resp.valid && Mux(valid, req.rw, io.req.valid && io.req.bits.rw) + + io.writeback.valid := io.req.valid && io.req.ready && io.req.bits.isWriteback + io.writeback.bits := io.req.bits.addr + + q.io.deq.ready := Mux(q.io.deq.bits.isWriteback, io.writeback_data.ready, Bool(true)) + io.resp.valid := q.io.deq.valid && !q.io.deq.bits.isWriteback + io.resp.bits := q.io.deq.bits + io.writeback_data.valid := q.io.deq.valid && q.io.deq.bits.isWriteback + io.writeback_data.bits := q.io.deq.bits +} + +class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits]) extends Component +{ + val io = new Bundle { + val cpu = new ioMem().flip + val mem = new ioMem + } + + val tagWidth = PADDR_BITS - OFFSET_BITS - log2Up(sets) + val metaWidth = tagWidth + 2 // valid + dirty + + val memCmdArb = (new Arbiter(2)) { new MemReqCmd } + val dataArb = (new Arbiter(2)) { new LLCDataReq(ways) } + val mshr = new LLCMSHRFile(sets, ways, outstanding) + val tags = new BigMem(sets, 2, tagLeaf)(Bits(width = metaWidth*ways)) + val data = new LLCData(sets, ways, dataLeaf) + val writeback = new LLCWriteback(2) + + val initCount = Reg(resetVal = UFix(0, log2Up(sets+1))) + val initialize = !initCount(log2Up(sets)) + when (initialize) { initCount := initCount + UFix(1) } + + val stall_s1 = Bool() + val replay_s1 = Reg(resetVal = Bool(false)) + val s1_valid = Reg(io.cpu.req_cmd.valid && !stall_s1 || replay_s1, resetVal = Bool(false)) + replay_s1 := s1_valid && stall_s1 + val s1 = Reg() { new MemReqCmd } + when (io.cpu.req_cmd.valid && io.cpu.req_cmd.ready) { s1 := io.cpu.req_cmd.bits } + + tags.io.en := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || mshr.io.tag.valid + tags.io.addr := Mux(initialize, initCount, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr))(log2Up(sets)-1,0)) + tags.io.rw := initialize || mshr.io.tag.valid + tags.io.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(Bool(false), Bool(true), mshr.io.tag.bits.addr(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) + tags.io.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(mshr.io.tag.bits.way))) + + val stall_s2 = Bool() + val s2_valid = Reg(resetVal = Bool(false)) + s2_valid := s1_valid && !replay_s1 && !stall_s1 || stall_s2 + val s2 = Reg() { new MemReqCmd } + when (s1_valid && !stall_s1 && !replay_s1) { s2 := s1 } + val s2_tags = Vec(ways) { Bits(width = metaWidth) } + for (i <- 0 until ways) s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) + val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.width-1, s2.addr.width-tagWidth) === t(tagWidth-1, 0)) + val s2_hit = s2_hits.reduceLeft(_||_) + stall_s1 := initialize || mshr.io.tag.valid || s2_valid && !s2_hit || stall_s2 + val repl_way = LFSR16(s2_valid)(log2Up(ways)-1, 0) + val repl_tag = s2_tags(repl_way).toUFix + + mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw + mshr.io.cpu.bits := s2 + mshr.io.repl_way := repl_way + mshr.io.repl_dirty := repl_tag(tagWidth).toBool + mshr.io.repl_tag := repl_tag + mshr.io.mem.resp := io.mem.resp + + data.io.req <> dataArb.io.out + data.io.mem_resp := io.mem.resp + data.io.mem_resp_set := mshr.io.mem_resp_set + data.io.mem_resp_way := mshr.io.mem_resp_way + data.io.req_data.bits := io.cpu.req_data.bits + + writeback.io.req(0) <> data.io.writeback + writeback.io.data(0) <> data.io.writeback_data + writeback.io.req(1).valid := s2_valid && !s2_hit && s2.rw + writeback.io.req(1).bits := s2.addr + writeback.io.data(1).valid := io.cpu.req_data.valid + writeback.io.data(1).bits := io.cpu.req_data.bits + data.io.req_data.valid := io.cpu.req_data.valid && !writeback.io.data(1).ready + + memCmdArb.io.in(0) <> mshr.io.mem.req_cmd + memCmdArb.io.in(1) <> writeback.io.mem.req_cmd + + dataArb.io.in(0) <> mshr.io.data + dataArb.io.in(1).valid := s2_valid && s2_hit + dataArb.io.in(1).bits := s2 + dataArb.io.in(1).bits.way := OHToUFix(s2_hits) + dataArb.io.in(1).bits.isWriteback := Bool(false) + + stall_s2 := s2_valid && !Mux(s2_hit, dataArb.io.in(1).ready, Mux(s2.rw, writeback.io.req(1).ready, mshr.io.cpu.ready)) + + io.cpu.resp <> data.io.resp + io.cpu.req_cmd.ready := !stall_s1 && !replay_s1 + io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready + io.mem.req_cmd <> memCmdArb.io.out + io.mem.req_data <> writeback.io.mem.req_data +} From 1ebfeeca8a0024e2dc2281dc926d1da6ba439e69 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 10 Jul 2012 05:23:29 -0700 Subject: [PATCH 099/374] add L2$ It still has performance bugs but no correctness bugs AFAIK. --- uncore/llc.scala | 392 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 392 insertions(+) create mode 100644 uncore/llc.scala diff --git a/uncore/llc.scala b/uncore/llc.scala new file mode 100644 index 00000000..71dfa536 --- /dev/null +++ b/uncore/llc.scala @@ -0,0 +1,392 @@ +package rocket + +import Chisel._ +import Node._ +import Constants._ + +class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component +{ + val io = new Bundle { + val addr = UFix(log2Up(n), INPUT) + val en = Bool(INPUT) + val rw = Bool(INPUT) + val wdata = gen.asInput + val wmask = gen.asInput + val rdata = gen.asOutput + } + require(readLatency >= 0 && readLatency <= 2) + val data = gen + val colMux = if (2*data.width <= leaf.data.width && n > leaf.n) 1 << math.floor(math.log(leaf.data.width/data.width)/math.log(2)).toInt else 1 + val nWide = if (data.width > leaf.data.width) 1+(data.width-1)/leaf.data.width else 1 + val nDeep = if (n > colMux*leaf.n) 1+(n-1)/(colMux*leaf.n) else 1 + if (nDeep > 1 || colMux > 1) + require(isPow2(n) && isPow2(leaf.n)) + + val idx = io.addr(log2Up(n/nDeep/colMux)-1, 0) + val rdataDeep = Vec(nDeep) { Bits() } + val rdataSel = Vec(nDeep) { Bool() } + val cond = Vec(nDeep) { Bool() } + val ren = Vec(nDeep) { Bool() } + val reg_ren = Vec(nDeep) { Reg() { Bool() } } + val reg2_ren = Vec(nDeep) { Reg() { Bool() } } + val reg_raddr = Vec(nDeep) { Reg() { UFix() } } + val reg2_raddr = Vec(nDeep) { Reg() { UFix() } } + val renOut = Vec(nDeep) { Bool() } + val raddrOut = Vec(nDeep) { UFix() } + val rdata = Vec(nDeep) { Vec(nWide) { Bits() } } + val wdata = io.wdata.toBits + val wmask = io.wmask.toBits + for (i <- 0 until nDeep) { + cond(i) := (if (nDeep == 1) io.en else io.en && UFix(i) === io.addr(log2Up(n)-1, log2Up(n/nDeep))) + ren(i) := cond(i) && !io.rw + reg_ren(i) := ren(i) + reg2_ren(i) := reg_ren(i) + when (ren(i)) { reg_raddr(i) := io.addr } + when (reg_ren(i)) { reg2_raddr(i) := reg_raddr(i) } + renOut(i) := (if (readLatency > 1) reg2_ren(i) else if (readLatency > 0) reg_ren(i) else ren(i)) + raddrOut(i) := (if (readLatency > 1) reg2_raddr(i) else if (readLatency > 0) reg_raddr(i) else io.addr) + + for (j <- 0 until nWide) { + val mem = leaf.clone + var dout: Bits = null + val dout1 = if (readLatency > 0) Reg() { Bits() } else null + + var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) + if (colMux > 1) + wmask0 = wmask0 & FillInterleaved(gen.width, UFixToOH(io.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) + val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) + when (cond(i)) { + when (io.rw) { mem.write(idx, wdata0, wmask0) } + .otherwise { if (readLatency > 0) dout1 := mem(idx) } + } + + if (readLatency == 0) { + dout = mem(idx) + } else if (readLatency == 1) { + dout = dout1 + } else { + val dout2 = Reg() { Bits() } + when (reg_ren(i)) { dout2 := dout1 } + dout = dout2 + } + + rdata(i)(j) := dout + } + val rdataWide = rdata(i).reduceLeft((x, y) => Cat(y, x)) + + var colMuxOut = rdataWide + if (colMux > 1) { + val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.width*(k+1)-1, gen.width*k))) { Bits() } + colMuxOut = colMuxIn(raddrOut(i)(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) + } + + rdataDeep(i) := colMuxOut + rdataSel(i) := renOut(i) + } + + io.rdata := Mux1H(rdataSel, rdataDeep) +} + +class LLCDataReq(ways: Int) extends MemReqCmd +{ + val way = UFix(width = log2Up(ways)) + val isWriteback = Bool() + + override def clone = new LLCDataReq(ways).asInstanceOf[this.type] +} + +class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component +{ + val io = new Bundle { + val cpu = (new FIFOIO) { new MemReqCmd }.flip + val repl_way = UFix(log2Up(ways), INPUT) + val repl_dirty = Bool(INPUT) + val repl_tag = UFix(PADDR_BITS - OFFSET_BITS - log2Up(sets), INPUT) + val data = (new FIFOIO) { new LLCDataReq(ways) } + val tag = (new PipeIO) { new Bundle { + val addr = UFix(width = PADDR_BITS - OFFSET_BITS) + val way = UFix(width = log2Up(ways)) + } } + val mem = new ioMem + val mem_resp_set = UFix(log2Up(sets), OUTPUT) + val mem_resp_way = UFix(log2Up(ways), OUTPUT) + } + + class MSHR extends Bundle { + val addr = UFix(width = PADDR_BITS - OFFSET_BITS) + val way = UFix(width = log2Up(ways)) + val tag = io.cpu.bits.tag.clone + val refilled = Bool() + val refillCount = UFix(width = log2Up(REFILL_CYCLES)) + val requested = Bool() + val old_dirty = Bool() + val old_tag = UFix(width = PADDR_BITS - OFFSET_BITS - log2Up(sets)) + + override def clone = new MSHR().asInstanceOf[this.type] + } + + val valid = Vec(outstanding) { Reg(resetVal = Bool(false)) } + val validBits = valid.toBits + val freeId = PriorityEncoder(~validBits) + val mshr = Vec(outstanding) { Reg() { new MSHR } } + when (io.cpu.valid && io.cpu.ready) { + valid(freeId) := Bool(true) + mshr(freeId).addr := io.cpu.bits.addr + mshr(freeId).tag := io.cpu.bits.tag + mshr(freeId).way := io.repl_way + mshr(freeId).old_dirty := io.repl_dirty + mshr(freeId).old_tag := io.repl_tag + mshr(freeId).requested := Bool(false) + mshr(freeId).refillCount := UFix(0) + mshr(freeId).refilled := Bool(false) + } + + val requests = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && !mshr(i).old_dirty && !mshr(i).requested):_*) + val request = requests.orR + val requestId = PriorityEncoder(requests) + when (io.mem.req_cmd.valid && io.mem.req_cmd.ready) { mshr(requestId).requested := Bool(true) } + + val refillId = io.mem.resp.bits.tag(log2Up(outstanding)-1, 0) + val refillCount = mshr(refillId).refillCount + when (io.mem.resp.valid) { + mshr(refillId).refillCount := refillCount + UFix(1) + when (refillCount === UFix(REFILL_CYCLES-1)) { mshr(refillId).refilled := Bool(true) } + } + + val replays = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).refilled):_*) + val replay = replays.orR + val replayId = PriorityEncoder(replays) + when (replay && io.data.ready) { valid(replayId) := Bool(false) } + + val writebacks = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).old_dirty):_*) + val writeback = writebacks.orR + val writebackId = PriorityEncoder(writebacks) + when (writeback && io.data.ready && !replay) { mshr(writebackId).old_dirty := Bool(false) } + + val conflicts = Cat(Bits(0), (0 until outstanding).map(i => valid(i) && io.cpu.bits.addr(log2Up(sets)-1, 0) === mshr(i).addr(log2Up(sets)-1, 0)):_*) + io.cpu.ready := !conflicts.orR && !validBits.andR + + io.data.valid := replay || writeback + io.data.bits.rw := Bool(false) + io.data.bits.tag := mshr(replayId).tag + io.data.bits.isWriteback := Bool(true) + io.data.bits.addr := Cat(mshr(writebackId).old_tag, mshr(writebackId).addr(log2Up(sets)-1, 0)).toUFix + io.data.bits.way := mshr(writebackId).way + when (replay) { + io.data.bits.isWriteback := Bool(false) + io.data.bits.addr := mshr(replayId).addr + io.data.bits.way := mshr(replayId).way + } + io.tag.valid := replay && io.data.ready + io.tag.bits.addr := io.data.bits.addr + io.tag.bits.way := io.data.bits.way + + io.mem.req_cmd.valid := request + io.mem.req_cmd.bits.rw := Bool(false) + io.mem.req_cmd.bits.addr := mshr(requestId).addr + io.mem.req_cmd.bits.tag := requestId + io.mem_resp_set := mshr(refillId).addr + io.mem_resp_way := mshr(refillId).way +} + +class LLCWriteback(requestors: Int) extends Component +{ + val io = new Bundle { + val req = Vec(requestors) { (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) }.flip } + val data = Vec(requestors) { (new FIFOIO) { new MemData }.flip } + val mem = new ioMem + } + + val valid = Reg(resetVal = Bool(false)) + val who = Reg() { UFix() } + val addr = Reg() { UFix() } + val cmd_sent = Reg() { Bool() } + val data_sent = Reg() { Bool() } + val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) + + var anyReq = Bool(false) + for (i <- 0 until requestors) { + io.req(i).ready := !valid && !anyReq + io.data(i).ready := valid && who === UFix(i) && io.mem.req_data.ready + anyReq = anyReq || io.req(i).valid + } + + val nextWho = PriorityEncoder(io.req.map(_.valid)) + when (!valid && io.req.map(_.valid).reduceLeft(_||_)) { + valid := Bool(true) + cmd_sent := Bool(false) + data_sent := Bool(false) + who := nextWho + addr := io.req(nextWho).bits + } + + when (io.mem.req_data.valid && io.mem.req_data.ready) { + count := count + UFix(1) + when (count === UFix(REFILL_CYCLES-1)) { + data_sent := Bool(true) + when (cmd_sent) { valid := Bool(false) } + } + } + when (io.mem.req_cmd.valid && io.mem.req_cmd.ready) { cmd_sent := Bool(true) } + when (valid && cmd_sent && data_sent) { valid := Bool(false) } + + io.mem.req_cmd.valid := valid && !cmd_sent + io.mem.req_cmd.bits.addr := addr + io.mem.req_cmd.bits.rw := Bool(true) + + io.mem.req_data.valid := valid && !data_sent && io.data(who).valid + io.mem.req_data.bits := io.data(who).bits +} + +class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component +{ + val io = new Bundle { + val req = (new FIFOIO) { new LLCDataReq(ways) }.flip + val req_data = (new FIFOIO) { new MemData }.flip + val writeback = (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) } + val writeback_data = (new FIFOIO) { new MemData } + val resp = (new PipeIO) { new MemResp } + val mem_resp = (new PipeIO) { new MemResp }.flip + val mem_resp_set = UFix(log2Up(sets), INPUT) + val mem_resp_way = UFix(log2Up(ways), INPUT) + } + + val data = new BigMem(sets*ways*REFILL_CYCLES, 2, leaf)(Bits(width = MEM_DATA_BITS)) + class QEntry extends MemResp { + val isWriteback = Bool() + override def clone = new QEntry().asInstanceOf[this.type] + } + val q = (new queue(4)) { new QEntry } + val qReady = q.io.count <= UFix(q.entries - 3) + val valid = Reg(resetVal = Bool(false)) + val req = Reg() { io.req.bits.clone } + val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) + val refillCount = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) + + when (data.io.en && !io.mem_resp.valid) { + count := count + UFix(1) + when (valid && count === UFix(REFILL_CYCLES-1)) { valid := Bool(false) } + } + when (io.req.valid && io.req.ready) { valid := Bool(true); req := io.req.bits } + when (io.mem_resp.valid) { refillCount := refillCount + UFix(1) } + + data.io.en := io.req.valid && io.req.ready && Mux(io.req.bits.rw, io.req_data.valid, qReady) + data.io.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUFix + data.io.rw := io.req.bits.rw + data.io.wdata := io.req_data.bits.data + data.io.wmask := Fix(-1, io.req_data.bits.data.width) + when (valid) { + data.io.en := Mux(req.rw, io.req_data.valid, qReady) + data.io.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUFix + data.io.rw := req.rw + } + when (io.mem_resp.valid) { + data.io.en := Bool(true) + data.io.addr := Cat(io.mem_resp_way, io.mem_resp_set, refillCount).toUFix + data.io.rw := Bool(true) + data.io.wdata := io.mem_resp.bits.data + } + + q.io.enq.valid := Reg(Reg(data.io.en && !data.io.rw, resetVal = Bool(false)), resetVal = Bool(false)) + q.io.enq.bits.tag := Reg(Reg(Mux(valid, req.tag, io.req.bits.tag))) + q.io.enq.bits.data := data.io.rdata + q.io.enq.bits.isWriteback := Reg(Reg(Mux(valid, req.isWriteback, io.req.bits.isWriteback))) + + io.req.ready := !valid && Mux(io.req.bits.isWriteback, io.writeback.ready, Bool(true)) + io.req_data.ready := !io.mem_resp.valid && Mux(valid, req.rw, io.req.valid && io.req.bits.rw) + + io.writeback.valid := io.req.valid && io.req.ready && io.req.bits.isWriteback + io.writeback.bits := io.req.bits.addr + + q.io.deq.ready := Mux(q.io.deq.bits.isWriteback, io.writeback_data.ready, Bool(true)) + io.resp.valid := q.io.deq.valid && !q.io.deq.bits.isWriteback + io.resp.bits := q.io.deq.bits + io.writeback_data.valid := q.io.deq.valid && q.io.deq.bits.isWriteback + io.writeback_data.bits := q.io.deq.bits +} + +class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits]) extends Component +{ + val io = new Bundle { + val cpu = new ioMem().flip + val mem = new ioMem + } + + val tagWidth = PADDR_BITS - OFFSET_BITS - log2Up(sets) + val metaWidth = tagWidth + 2 // valid + dirty + + val memCmdArb = (new Arbiter(2)) { new MemReqCmd } + val dataArb = (new Arbiter(2)) { new LLCDataReq(ways) } + val mshr = new LLCMSHRFile(sets, ways, outstanding) + val tags = new BigMem(sets, 2, tagLeaf)(Bits(width = metaWidth*ways)) + val data = new LLCData(sets, ways, dataLeaf) + val writeback = new LLCWriteback(2) + + val initCount = Reg(resetVal = UFix(0, log2Up(sets+1))) + val initialize = !initCount(log2Up(sets)) + when (initialize) { initCount := initCount + UFix(1) } + + val stall_s1 = Bool() + val replay_s1 = Reg(resetVal = Bool(false)) + val s1_valid = Reg(io.cpu.req_cmd.valid && !stall_s1 || replay_s1, resetVal = Bool(false)) + replay_s1 := s1_valid && stall_s1 + val s1 = Reg() { new MemReqCmd } + when (io.cpu.req_cmd.valid && io.cpu.req_cmd.ready) { s1 := io.cpu.req_cmd.bits } + + tags.io.en := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || mshr.io.tag.valid + tags.io.addr := Mux(initialize, initCount, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr))(log2Up(sets)-1,0)) + tags.io.rw := initialize || mshr.io.tag.valid + tags.io.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(Bool(false), Bool(true), mshr.io.tag.bits.addr(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) + tags.io.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(mshr.io.tag.bits.way))) + + val stall_s2 = Bool() + val s2_valid = Reg(resetVal = Bool(false)) + s2_valid := s1_valid && !replay_s1 && !stall_s1 || stall_s2 + val s2 = Reg() { new MemReqCmd } + when (s1_valid && !stall_s1 && !replay_s1) { s2 := s1 } + val s2_tags = Vec(ways) { Bits(width = metaWidth) } + for (i <- 0 until ways) s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) + val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.width-1, s2.addr.width-tagWidth) === t(tagWidth-1, 0)) + val s2_hit = s2_hits.reduceLeft(_||_) + stall_s1 := initialize || mshr.io.tag.valid || s2_valid && !s2_hit || stall_s2 + val repl_way = LFSR16(s2_valid)(log2Up(ways)-1, 0) + val repl_tag = s2_tags(repl_way).toUFix + + mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw + mshr.io.cpu.bits := s2 + mshr.io.repl_way := repl_way + mshr.io.repl_dirty := repl_tag(tagWidth).toBool + mshr.io.repl_tag := repl_tag + mshr.io.mem.resp := io.mem.resp + + data.io.req <> dataArb.io.out + data.io.mem_resp := io.mem.resp + data.io.mem_resp_set := mshr.io.mem_resp_set + data.io.mem_resp_way := mshr.io.mem_resp_way + data.io.req_data.bits := io.cpu.req_data.bits + + writeback.io.req(0) <> data.io.writeback + writeback.io.data(0) <> data.io.writeback_data + writeback.io.req(1).valid := s2_valid && !s2_hit && s2.rw + writeback.io.req(1).bits := s2.addr + writeback.io.data(1).valid := io.cpu.req_data.valid + writeback.io.data(1).bits := io.cpu.req_data.bits + data.io.req_data.valid := io.cpu.req_data.valid && !writeback.io.data(1).ready + + memCmdArb.io.in(0) <> mshr.io.mem.req_cmd + memCmdArb.io.in(1) <> writeback.io.mem.req_cmd + + dataArb.io.in(0) <> mshr.io.data + dataArb.io.in(1).valid := s2_valid && s2_hit + dataArb.io.in(1).bits := s2 + dataArb.io.in(1).bits.way := OHToUFix(s2_hits) + dataArb.io.in(1).bits.isWriteback := Bool(false) + + stall_s2 := s2_valid && !Mux(s2_hit, dataArb.io.in(1).ready, Mux(s2.rw, writeback.io.req(1).ready, mshr.io.cpu.ready)) + + io.cpu.resp <> data.io.resp + io.cpu.req_cmd.ready := !stall_s1 && !replay_s1 + io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready + io.mem.req_cmd <> memCmdArb.io.out + io.mem.req_data <> writeback.io.mem.req_data +} From 0aa33bf9095ddc6b8980a7f12046fbf42bfbf230 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 11 Jul 2012 17:56:39 -0700 Subject: [PATCH 100/374] fix some LLC bugs --- uncore/llc.scala | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 71dfa536..e6f39c4a 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -103,7 +103,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val repl_dirty = Bool(INPUT) val repl_tag = UFix(PADDR_BITS - OFFSET_BITS - log2Up(sets), INPUT) val data = (new FIFOIO) { new LLCDataReq(ways) } - val tag = (new PipeIO) { new Bundle { + val tag = (new FIFOIO) { new Bundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) val way = UFix(width = log2Up(ways)) } } @@ -156,7 +156,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val replays = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).refilled):_*) val replay = replays.orR val replayId = PriorityEncoder(replays) - when (replay && io.data.ready) { valid(replayId) := Bool(false) } + when (replay && io.data.ready && io.tag.ready) { valid(replayId) := Bool(false) } val writebacks = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).old_dirty):_*) val writeback = writebacks.orR @@ -166,7 +166,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val conflicts = Cat(Bits(0), (0 until outstanding).map(i => valid(i) && io.cpu.bits.addr(log2Up(sets)-1, 0) === mshr(i).addr(log2Up(sets)-1, 0)):_*) io.cpu.ready := !conflicts.orR && !validBits.andR - io.data.valid := replay || writeback + io.data.valid := replay && io.tag.ready || writeback io.data.bits.rw := Bool(false) io.data.bits.tag := mshr(replayId).tag io.data.bits.isWriteback := Bool(true) @@ -318,7 +318,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val memCmdArb = (new Arbiter(2)) { new MemReqCmd } val dataArb = (new Arbiter(2)) { new LLCDataReq(ways) } val mshr = new LLCMSHRFile(sets, ways, outstanding) - val tags = new BigMem(sets, 2, tagLeaf)(Bits(width = metaWidth*ways)) + val tags = new BigMem(sets, 1, tagLeaf)(Bits(width = metaWidth*ways)) val data = new LLCData(sets, ways, dataLeaf) val writeback = new LLCWriteback(2) @@ -333,31 +333,38 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val s1 = Reg() { new MemReqCmd } when (io.cpu.req_cmd.valid && io.cpu.req_cmd.ready) { s1 := io.cpu.req_cmd.bits } - tags.io.en := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || mshr.io.tag.valid - tags.io.addr := Mux(initialize, initCount, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr))(log2Up(sets)-1,0)) - tags.io.rw := initialize || mshr.io.tag.valid - tags.io.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(Bool(false), Bool(true), mshr.io.tag.bits.addr(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) - tags.io.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(mshr.io.tag.bits.way))) - val stall_s2 = Bool() val s2_valid = Reg(resetVal = Bool(false)) s2_valid := s1_valid && !replay_s1 && !stall_s1 || stall_s2 val s2 = Reg() { new MemReqCmd } - when (s1_valid && !stall_s1 && !replay_s1) { s2 := s1 } - val s2_tags = Vec(ways) { Bits(width = metaWidth) } - for (i <- 0 until ways) s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) + val s2_tags = Vec(ways) { Reg() { Bits(width = metaWidth) } } + when (s1_valid && !stall_s1 && !replay_s1) { + s2 := s1 + for (i <- 0 until ways) + s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) + } val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.width-1, s2.addr.width-tagWidth) === t(tagWidth-1, 0)) + val s2_hit_way = OHToUFix(s2_hits) val s2_hit = s2_hits.reduceLeft(_||_) - stall_s1 := initialize || mshr.io.tag.valid || s2_valid && !s2_hit || stall_s2 + val s2_hit_dirty = s2_tags(s2_hit_way)(tagWidth+1) val repl_way = LFSR16(s2_valid)(log2Up(ways)-1, 0) val repl_tag = s2_tags(repl_way).toUFix + val setDirty = s2_valid && s2.rw && s2_hit && !s2_hit_dirty + stall_s1 := initialize || mshr.io.tag.valid || setDirty || s2_valid && !s2_hit || stall_s2 + + tags.io.en := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || setDirty || mshr.io.tag.valid + tags.io.addr := Mux(initialize, initCount, Mux(setDirty, s2.addr, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)))(log2Up(sets)-1,0)) + tags.io.rw := initialize || setDirty || mshr.io.tag.valid + tags.io.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) + tags.io.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way)))) mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw mshr.io.cpu.bits := s2 mshr.io.repl_way := repl_way - mshr.io.repl_dirty := repl_tag(tagWidth).toBool + mshr.io.repl_dirty := repl_tag(tagWidth+1, tagWidth).andR mshr.io.repl_tag := repl_tag mshr.io.mem.resp := io.mem.resp + mshr.io.tag.ready := !setDirty data.io.req <> dataArb.io.out data.io.mem_resp := io.mem.resp @@ -379,7 +386,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da dataArb.io.in(0) <> mshr.io.data dataArb.io.in(1).valid := s2_valid && s2_hit dataArb.io.in(1).bits := s2 - dataArb.io.in(1).bits.way := OHToUFix(s2_hits) + dataArb.io.in(1).bits.way := s2_hit_way dataArb.io.in(1).bits.isWriteback := Bool(false) stall_s2 := s2_valid && !Mux(s2_hit, dataArb.io.in(1).ready, Mux(s2.rw, writeback.io.req(1).ready, mshr.io.cpu.ready)) From 62a3ea411385f65e8984acc14198b143f0606be1 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 11 Jul 2012 17:56:39 -0700 Subject: [PATCH 101/374] fix some LLC bugs --- uncore/llc.scala | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 71dfa536..e6f39c4a 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -103,7 +103,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val repl_dirty = Bool(INPUT) val repl_tag = UFix(PADDR_BITS - OFFSET_BITS - log2Up(sets), INPUT) val data = (new FIFOIO) { new LLCDataReq(ways) } - val tag = (new PipeIO) { new Bundle { + val tag = (new FIFOIO) { new Bundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) val way = UFix(width = log2Up(ways)) } } @@ -156,7 +156,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val replays = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).refilled):_*) val replay = replays.orR val replayId = PriorityEncoder(replays) - when (replay && io.data.ready) { valid(replayId) := Bool(false) } + when (replay && io.data.ready && io.tag.ready) { valid(replayId) := Bool(false) } val writebacks = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).old_dirty):_*) val writeback = writebacks.orR @@ -166,7 +166,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val conflicts = Cat(Bits(0), (0 until outstanding).map(i => valid(i) && io.cpu.bits.addr(log2Up(sets)-1, 0) === mshr(i).addr(log2Up(sets)-1, 0)):_*) io.cpu.ready := !conflicts.orR && !validBits.andR - io.data.valid := replay || writeback + io.data.valid := replay && io.tag.ready || writeback io.data.bits.rw := Bool(false) io.data.bits.tag := mshr(replayId).tag io.data.bits.isWriteback := Bool(true) @@ -318,7 +318,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val memCmdArb = (new Arbiter(2)) { new MemReqCmd } val dataArb = (new Arbiter(2)) { new LLCDataReq(ways) } val mshr = new LLCMSHRFile(sets, ways, outstanding) - val tags = new BigMem(sets, 2, tagLeaf)(Bits(width = metaWidth*ways)) + val tags = new BigMem(sets, 1, tagLeaf)(Bits(width = metaWidth*ways)) val data = new LLCData(sets, ways, dataLeaf) val writeback = new LLCWriteback(2) @@ -333,31 +333,38 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val s1 = Reg() { new MemReqCmd } when (io.cpu.req_cmd.valid && io.cpu.req_cmd.ready) { s1 := io.cpu.req_cmd.bits } - tags.io.en := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || mshr.io.tag.valid - tags.io.addr := Mux(initialize, initCount, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr))(log2Up(sets)-1,0)) - tags.io.rw := initialize || mshr.io.tag.valid - tags.io.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(Bool(false), Bool(true), mshr.io.tag.bits.addr(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) - tags.io.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(mshr.io.tag.bits.way))) - val stall_s2 = Bool() val s2_valid = Reg(resetVal = Bool(false)) s2_valid := s1_valid && !replay_s1 && !stall_s1 || stall_s2 val s2 = Reg() { new MemReqCmd } - when (s1_valid && !stall_s1 && !replay_s1) { s2 := s1 } - val s2_tags = Vec(ways) { Bits(width = metaWidth) } - for (i <- 0 until ways) s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) + val s2_tags = Vec(ways) { Reg() { Bits(width = metaWidth) } } + when (s1_valid && !stall_s1 && !replay_s1) { + s2 := s1 + for (i <- 0 until ways) + s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) + } val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.width-1, s2.addr.width-tagWidth) === t(tagWidth-1, 0)) + val s2_hit_way = OHToUFix(s2_hits) val s2_hit = s2_hits.reduceLeft(_||_) - stall_s1 := initialize || mshr.io.tag.valid || s2_valid && !s2_hit || stall_s2 + val s2_hit_dirty = s2_tags(s2_hit_way)(tagWidth+1) val repl_way = LFSR16(s2_valid)(log2Up(ways)-1, 0) val repl_tag = s2_tags(repl_way).toUFix + val setDirty = s2_valid && s2.rw && s2_hit && !s2_hit_dirty + stall_s1 := initialize || mshr.io.tag.valid || setDirty || s2_valid && !s2_hit || stall_s2 + + tags.io.en := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || setDirty || mshr.io.tag.valid + tags.io.addr := Mux(initialize, initCount, Mux(setDirty, s2.addr, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)))(log2Up(sets)-1,0)) + tags.io.rw := initialize || setDirty || mshr.io.tag.valid + tags.io.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) + tags.io.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way)))) mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw mshr.io.cpu.bits := s2 mshr.io.repl_way := repl_way - mshr.io.repl_dirty := repl_tag(tagWidth).toBool + mshr.io.repl_dirty := repl_tag(tagWidth+1, tagWidth).andR mshr.io.repl_tag := repl_tag mshr.io.mem.resp := io.mem.resp + mshr.io.tag.ready := !setDirty data.io.req <> dataArb.io.out data.io.mem_resp := io.mem.resp @@ -379,7 +386,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da dataArb.io.in(0) <> mshr.io.data dataArb.io.in(1).valid := s2_valid && s2_hit dataArb.io.in(1).bits := s2 - dataArb.io.in(1).bits.way := OHToUFix(s2_hits) + dataArb.io.in(1).bits.way := s2_hit_way dataArb.io.in(1).bits.isWriteback := Bool(false) stall_s2 := s2_valid && !Mux(s2_hit, dataArb.io.in(1).ready, Mux(s2.rw, writeback.io.req(1).ready, mshr.io.cpu.ready)) From 18bc14058b3e21ef58b45e59039fd65733f2fd83 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Thu, 12 Jul 2012 18:12:49 -0700 Subject: [PATCH 102/374] INPUT/OUTPUT orderring swapped --- uncore/llc.scala | 14 +++++++------- uncore/uncore.scala | 30 +++++++++++++++--------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index e6f39c4a..5f5007d1 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -7,7 +7,7 @@ import Constants._ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component { val io = new Bundle { - val addr = UFix(log2Up(n), INPUT) + val addr = UFix(INPUT, log2Up(n)) val en = Bool(INPUT) val rw = Bool(INPUT) val wdata = gen.asInput @@ -99,17 +99,17 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component { val io = new Bundle { val cpu = (new FIFOIO) { new MemReqCmd }.flip - val repl_way = UFix(log2Up(ways), INPUT) + val repl_way = UFix(INPUT, log2Up(ways)) val repl_dirty = Bool(INPUT) - val repl_tag = UFix(PADDR_BITS - OFFSET_BITS - log2Up(sets), INPUT) + val repl_tag = UFix(INPUT, PADDR_BITS - OFFSET_BITS - log2Up(sets)) val data = (new FIFOIO) { new LLCDataReq(ways) } val tag = (new FIFOIO) { new Bundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) val way = UFix(width = log2Up(ways)) } } val mem = new ioMem - val mem_resp_set = UFix(log2Up(sets), OUTPUT) - val mem_resp_way = UFix(log2Up(ways), OUTPUT) + val mem_resp_set = UFix(OUTPUT, log2Up(sets)) + val mem_resp_way = UFix(OUTPUT, log2Up(ways)) } class MSHR extends Bundle { @@ -247,8 +247,8 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component val writeback_data = (new FIFOIO) { new MemData } val resp = (new PipeIO) { new MemResp } val mem_resp = (new PipeIO) { new MemResp }.flip - val mem_resp_set = UFix(log2Up(sets), INPUT) - val mem_resp_way = UFix(log2Up(ways), INPUT) + val mem_resp_set = UFix(INPUT, log2Up(sets)) + val mem_resp_way = UFix(INPUT, log2Up(ways)) } val data = new BigMem(sets*ways*REFILL_CYCLES, 2, leaf)(Bits(width = MEM_DATA_BITS)) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 6239981f..7e34492c 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -56,8 +56,8 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val p_data = (new PipeIO) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) - val p_rep_cnt_dec = Bits(ntiles, INPUT) - val p_req_cnt_inc = Bits(ntiles, INPUT) + val p_rep_cnt_dec = Bits(INPUT, ntiles) + val p_req_cnt_inc = Bits(INPUT, ntiles) val p_rep_data = (new PipeIO) { new ProbeReplyData }.flip val x_init_data = (new PipeIO) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) @@ -69,19 +69,19 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val mem_req_lock = Bool(OUTPUT) val probe_req = (new FIFOIO) { new ProbeRequest } val busy = Bool(OUTPUT) - val addr = Bits(PADDR_BITS - OFFSET_BITS, OUTPUT) - val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) - val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) - val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) - val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) - val x_type = Bits(X_INIT_TYPE_MAX_BITS, OUTPUT) - val push_p_req = Bits(ntiles, OUTPUT) - val pop_p_rep = Bits(ntiles, OUTPUT) - val pop_p_rep_data = Bits(ntiles, OUTPUT) - val pop_p_rep_dep = Bits(ntiles, OUTPUT) - val pop_x_init = Bits(ntiles, OUTPUT) - val pop_x_init_data = Bits(ntiles, OUTPUT) - val pop_x_init_dep = Bits(ntiles, OUTPUT) + val addr = Bits(OUTPUT, PADDR_BITS - OFFSET_BITS) + val init_tile_id = Bits(OUTPUT, TILE_ID_BITS) + val p_rep_tile_id = Bits(OUTPUT, TILE_ID_BITS) + val tile_xact_id = Bits(OUTPUT, TILE_XACT_ID_BITS) + val sharer_count = Bits(OUTPUT, TILE_ID_BITS+1) + val x_type = Bits(OUTPUT, X_INIT_TYPE_MAX_BITS) + val push_p_req = Bits(OUTPUT, ntiles) + val pop_p_rep = Bits(OUTPUT, ntiles) + val pop_p_rep_data = Bits(OUTPUT, ntiles) + val pop_p_rep_dep = Bits(OUTPUT, ntiles) + val pop_x_init = Bits(OUTPUT, ntiles) + val pop_x_init_data = Bits(OUTPUT, ntiles) + val pop_x_init_dep = Bits(OUTPUT, ntiles) val send_x_rep_ack = Bool(OUTPUT) } From a79747a06221802ac05c65144d3b8b42d5b7adb2 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Thu, 12 Jul 2012 18:12:49 -0700 Subject: [PATCH 103/374] INPUT/OUTPUT orderring swapped --- uncore/llc.scala | 14 +++++++------- uncore/uncore.scala | 30 +++++++++++++++--------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index e6f39c4a..5f5007d1 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -7,7 +7,7 @@ import Constants._ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component { val io = new Bundle { - val addr = UFix(log2Up(n), INPUT) + val addr = UFix(INPUT, log2Up(n)) val en = Bool(INPUT) val rw = Bool(INPUT) val wdata = gen.asInput @@ -99,17 +99,17 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component { val io = new Bundle { val cpu = (new FIFOIO) { new MemReqCmd }.flip - val repl_way = UFix(log2Up(ways), INPUT) + val repl_way = UFix(INPUT, log2Up(ways)) val repl_dirty = Bool(INPUT) - val repl_tag = UFix(PADDR_BITS - OFFSET_BITS - log2Up(sets), INPUT) + val repl_tag = UFix(INPUT, PADDR_BITS - OFFSET_BITS - log2Up(sets)) val data = (new FIFOIO) { new LLCDataReq(ways) } val tag = (new FIFOIO) { new Bundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) val way = UFix(width = log2Up(ways)) } } val mem = new ioMem - val mem_resp_set = UFix(log2Up(sets), OUTPUT) - val mem_resp_way = UFix(log2Up(ways), OUTPUT) + val mem_resp_set = UFix(OUTPUT, log2Up(sets)) + val mem_resp_way = UFix(OUTPUT, log2Up(ways)) } class MSHR extends Bundle { @@ -247,8 +247,8 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component val writeback_data = (new FIFOIO) { new MemData } val resp = (new PipeIO) { new MemResp } val mem_resp = (new PipeIO) { new MemResp }.flip - val mem_resp_set = UFix(log2Up(sets), INPUT) - val mem_resp_way = UFix(log2Up(ways), INPUT) + val mem_resp_set = UFix(INPUT, log2Up(sets)) + val mem_resp_way = UFix(INPUT, log2Up(ways)) } val data = new BigMem(sets*ways*REFILL_CYCLES, 2, leaf)(Bits(width = MEM_DATA_BITS)) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 6239981f..7e34492c 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -56,8 +56,8 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val p_data = (new PipeIO) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) - val p_rep_cnt_dec = Bits(ntiles, INPUT) - val p_req_cnt_inc = Bits(ntiles, INPUT) + val p_rep_cnt_dec = Bits(INPUT, ntiles) + val p_req_cnt_inc = Bits(INPUT, ntiles) val p_rep_data = (new PipeIO) { new ProbeReplyData }.flip val x_init_data = (new PipeIO) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) @@ -69,19 +69,19 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val mem_req_lock = Bool(OUTPUT) val probe_req = (new FIFOIO) { new ProbeRequest } val busy = Bool(OUTPUT) - val addr = Bits(PADDR_BITS - OFFSET_BITS, OUTPUT) - val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) - val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) - val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) - val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) - val x_type = Bits(X_INIT_TYPE_MAX_BITS, OUTPUT) - val push_p_req = Bits(ntiles, OUTPUT) - val pop_p_rep = Bits(ntiles, OUTPUT) - val pop_p_rep_data = Bits(ntiles, OUTPUT) - val pop_p_rep_dep = Bits(ntiles, OUTPUT) - val pop_x_init = Bits(ntiles, OUTPUT) - val pop_x_init_data = Bits(ntiles, OUTPUT) - val pop_x_init_dep = Bits(ntiles, OUTPUT) + val addr = Bits(OUTPUT, PADDR_BITS - OFFSET_BITS) + val init_tile_id = Bits(OUTPUT, TILE_ID_BITS) + val p_rep_tile_id = Bits(OUTPUT, TILE_ID_BITS) + val tile_xact_id = Bits(OUTPUT, TILE_XACT_ID_BITS) + val sharer_count = Bits(OUTPUT, TILE_ID_BITS+1) + val x_type = Bits(OUTPUT, X_INIT_TYPE_MAX_BITS) + val push_p_req = Bits(OUTPUT, ntiles) + val pop_p_rep = Bits(OUTPUT, ntiles) + val pop_p_rep_data = Bits(OUTPUT, ntiles) + val pop_p_rep_dep = Bits(OUTPUT, ntiles) + val pop_x_init = Bits(OUTPUT, ntiles) + val pop_x_init_data = Bits(OUTPUT, ntiles) + val pop_x_init_dep = Bits(OUTPUT, ntiles) val send_x_rep_ack = Bool(OUTPUT) } From 0258dfb23f28e109dbcd7032e1f5f64aa03d2c5f Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 17 Jul 2012 22:55:00 -0700 Subject: [PATCH 104/374] decouple all interfaces between tile and top also, add an "incoherent" bit to tilelink to indicate no probes needed --- uncore/llc.scala | 10 +++++----- uncore/uncore.scala | 17 +++++++++++++---- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 5f5007d1..1c9d49b8 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -107,7 +107,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val addr = UFix(width = PADDR_BITS - OFFSET_BITS) val way = UFix(width = log2Up(ways)) } } - val mem = new ioMem + val mem = new ioMemPipe val mem_resp_set = UFix(OUTPUT, log2Up(sets)) val mem_resp_way = UFix(OUTPUT, log2Up(ways)) } @@ -194,7 +194,7 @@ class LLCWriteback(requestors: Int) extends Component val io = new Bundle { val req = Vec(requestors) { (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) }.flip } val data = Vec(requestors) { (new FIFOIO) { new MemData }.flip } - val mem = new ioMem + val mem = new ioMemPipe } val valid = Reg(resetVal = Bool(false)) @@ -245,7 +245,7 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component val req_data = (new FIFOIO) { new MemData }.flip val writeback = (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) } val writeback_data = (new FIFOIO) { new MemData } - val resp = (new PipeIO) { new MemResp } + val resp = (new FIFOIO) { new MemResp } val mem_resp = (new PipeIO) { new MemResp }.flip val mem_resp_set = UFix(INPUT, log2Up(sets)) val mem_resp_way = UFix(INPUT, log2Up(ways)) @@ -298,7 +298,7 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component io.writeback.valid := io.req.valid && io.req.ready && io.req.bits.isWriteback io.writeback.bits := io.req.bits.addr - q.io.deq.ready := Mux(q.io.deq.bits.isWriteback, io.writeback_data.ready, Bool(true)) + q.io.deq.ready := Mux(q.io.deq.bits.isWriteback, io.writeback_data.ready, io.resp.ready) io.resp.valid := q.io.deq.valid && !q.io.deq.bits.isWriteback io.resp.bits := q.io.deq.bits io.writeback_data.valid := q.io.deq.valid && q.io.deq.bits.isWriteback @@ -309,7 +309,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da { val io = new Bundle { val cpu = new ioMem().flip - val mem = new ioMem + val mem = new ioMemPipe } val tagWidth = PADDR_BITS - OFFSET_BITS - log2Up(sets) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 7e34492c..ead75252 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -20,6 +20,13 @@ class MemResp () extends MemData } class ioMem() extends Bundle +{ + val req_cmd = (new FIFOIO) { new MemReqCmd() } + val req_data = (new FIFOIO) { new MemData() } + val resp = (new FIFOIO) { new MemResp() }.flip +} + +class ioMemPipe() extends Bundle { val req_cmd = (new FIFOIO) { new MemReqCmd() } val req_data = (new FIFOIO) { new MemData() } @@ -46,8 +53,9 @@ class ioTileLink extends Bundle { val probe_req = (new FIFOIO) { new ProbeRequest }.flip val probe_rep = (new FIFOIO) { new ProbeReply } val probe_rep_data = (new FIFOIO) { new ProbeReplyData } - val xact_rep = (new PipeIO) { new TransactionReply }.flip + val xact_rep = (new FIFOIO) { new TransactionReply }.flip val xact_finish = (new FIFOIO) { new TransactionFinish } + val incoherent = Bool(OUTPUT) } class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { @@ -58,6 +66,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val xact_finish = Bool(INPUT) val p_rep_cnt_dec = Bits(INPUT, ntiles) val p_req_cnt_inc = Bits(INPUT, ntiles) + val tile_incoherent = Bits(INPUT, ntiles) val p_rep_data = (new PipeIO) { new ProbeReplyData }.flip val x_init_data = (new PipeIO) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) @@ -169,7 +178,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) - val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only + val p_req_initial_flags = ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id)) //TODO: Broadcast only p_req_flags := p_req_initial_flags(ntiles-1,0) mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) @@ -310,6 +319,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH t.p_data.valid := p_data_valid_arr(i) t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits + t.tile_incoherent := (Vec(io.tiles.map(_.incoherent)) { Bool() }).toBits t.sent_x_rep_ack := sent_x_rep_ack_arr(i) do_free_arr(i) := Bool(false) sent_x_rep_ack_arr(i) := Bool(false) @@ -360,8 +370,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH } } } - // If there were a ready signal due to e.g. intervening network use: - //io.mem.resp.ready := io.tiles(init_tile_id_arr.read(mem_idx)).xact_rep.ready + io.mem.resp.ready := io.tiles(init_tile_id_arr(mem_idx)).xact_rep.ready // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests From d01e70c6720310386c7c3f8708d24db7798da72d Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 17 Jul 2012 22:55:00 -0700 Subject: [PATCH 105/374] decouple all interfaces between tile and top also, add an "incoherent" bit to tilelink to indicate no probes needed --- uncore/llc.scala | 10 +++++----- uncore/uncore.scala | 17 +++++++++++++---- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 5f5007d1..1c9d49b8 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -107,7 +107,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val addr = UFix(width = PADDR_BITS - OFFSET_BITS) val way = UFix(width = log2Up(ways)) } } - val mem = new ioMem + val mem = new ioMemPipe val mem_resp_set = UFix(OUTPUT, log2Up(sets)) val mem_resp_way = UFix(OUTPUT, log2Up(ways)) } @@ -194,7 +194,7 @@ class LLCWriteback(requestors: Int) extends Component val io = new Bundle { val req = Vec(requestors) { (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) }.flip } val data = Vec(requestors) { (new FIFOIO) { new MemData }.flip } - val mem = new ioMem + val mem = new ioMemPipe } val valid = Reg(resetVal = Bool(false)) @@ -245,7 +245,7 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component val req_data = (new FIFOIO) { new MemData }.flip val writeback = (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) } val writeback_data = (new FIFOIO) { new MemData } - val resp = (new PipeIO) { new MemResp } + val resp = (new FIFOIO) { new MemResp } val mem_resp = (new PipeIO) { new MemResp }.flip val mem_resp_set = UFix(INPUT, log2Up(sets)) val mem_resp_way = UFix(INPUT, log2Up(ways)) @@ -298,7 +298,7 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component io.writeback.valid := io.req.valid && io.req.ready && io.req.bits.isWriteback io.writeback.bits := io.req.bits.addr - q.io.deq.ready := Mux(q.io.deq.bits.isWriteback, io.writeback_data.ready, Bool(true)) + q.io.deq.ready := Mux(q.io.deq.bits.isWriteback, io.writeback_data.ready, io.resp.ready) io.resp.valid := q.io.deq.valid && !q.io.deq.bits.isWriteback io.resp.bits := q.io.deq.bits io.writeback_data.valid := q.io.deq.valid && q.io.deq.bits.isWriteback @@ -309,7 +309,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da { val io = new Bundle { val cpu = new ioMem().flip - val mem = new ioMem + val mem = new ioMemPipe } val tagWidth = PADDR_BITS - OFFSET_BITS - log2Up(sets) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 7e34492c..ead75252 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -20,6 +20,13 @@ class MemResp () extends MemData } class ioMem() extends Bundle +{ + val req_cmd = (new FIFOIO) { new MemReqCmd() } + val req_data = (new FIFOIO) { new MemData() } + val resp = (new FIFOIO) { new MemResp() }.flip +} + +class ioMemPipe() extends Bundle { val req_cmd = (new FIFOIO) { new MemReqCmd() } val req_data = (new FIFOIO) { new MemData() } @@ -46,8 +53,9 @@ class ioTileLink extends Bundle { val probe_req = (new FIFOIO) { new ProbeRequest }.flip val probe_rep = (new FIFOIO) { new ProbeReply } val probe_rep_data = (new FIFOIO) { new ProbeReplyData } - val xact_rep = (new PipeIO) { new TransactionReply }.flip + val xact_rep = (new FIFOIO) { new TransactionReply }.flip val xact_finish = (new FIFOIO) { new TransactionFinish } + val incoherent = Bool(OUTPUT) } class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { @@ -58,6 +66,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val xact_finish = Bool(INPUT) val p_rep_cnt_dec = Bits(INPUT, ntiles) val p_req_cnt_inc = Bits(INPUT, ntiles) + val tile_incoherent = Bits(INPUT, ntiles) val p_rep_data = (new PipeIO) { new ProbeReplyData }.flip val x_init_data = (new PipeIO) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) @@ -169,7 +178,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) - val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only + val p_req_initial_flags = ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id)) //TODO: Broadcast only p_req_flags := p_req_initial_flags(ntiles-1,0) mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) @@ -310,6 +319,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH t.p_data.valid := p_data_valid_arr(i) t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits + t.tile_incoherent := (Vec(io.tiles.map(_.incoherent)) { Bool() }).toBits t.sent_x_rep_ack := sent_x_rep_ack_arr(i) do_free_arr(i) := Bool(false) sent_x_rep_ack_arr(i) := Bool(false) @@ -360,8 +370,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH } } } - // If there were a ready signal due to e.g. intervening network use: - //io.mem.resp.ready := io.tiles(init_tile_id_arr.read(mem_idx)).xact_rep.ready + io.mem.resp.ready := io.tiles(init_tile_id_arr(mem_idx)).xact_rep.ready // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests From c6ac83658130d6c8c80f234aaad939eb80f80875 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sun, 22 Jul 2012 21:05:52 -0700 Subject: [PATCH 106/374] don't dequeue probe queue during reset --- uncore/uncore.scala | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index ead75252..6ea359ce 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -140,6 +140,8 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) + val p_req_initial_flags = Bits(width = ntiles) + p_req_initial_flags := ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id(log2Up(ntiles)-1,0))) //TODO: Broadcast only io.busy := state != s_idle io.addr := addr_ @@ -178,15 +180,14 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) - val p_req_initial_flags = ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id)) //TODO: Broadcast only - p_req_flags := p_req_initial_flags(ntiles-1,0) + p_req_flags := p_req_initial_flags mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id if(ntiles > 1) { - p_rep_count := UFix(ntiles-1) - state := Mux(p_req_initial_flags(ntiles-1,0).orR, s_probe, s_mem) + p_rep_count := PopCount(p_req_initial_flags) + state := Mux(p_req_initial_flags.orR, s_probe, s_mem) } else state := s_mem } } From df8aff090672eb388bf165fc85c45e5ebd7fd0fd Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sun, 22 Jul 2012 21:05:52 -0700 Subject: [PATCH 107/374] don't dequeue probe queue during reset --- uncore/uncore.scala | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index ead75252..6ea359ce 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -140,6 +140,8 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) + val p_req_initial_flags = Bits(width = ntiles) + p_req_initial_flags := ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id(log2Up(ntiles)-1,0))) //TODO: Broadcast only io.busy := state != s_idle io.addr := addr_ @@ -178,15 +180,14 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) - val p_req_initial_flags = ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id)) //TODO: Broadcast only - p_req_flags := p_req_initial_flags(ntiles-1,0) + p_req_flags := p_req_initial_flags mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id if(ntiles > 1) { - p_rep_count := UFix(ntiles-1) - state := Mux(p_req_initial_flags(ntiles-1,0).orR, s_probe, s_mem) + p_rep_count := PopCount(p_req_initial_flags) + state := Mux(p_req_initial_flags.orR, s_probe, s_mem) } else state := s_mem } } From d0e12c13f669ff1c1d3c3562dbcfe3bd98366b2d Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Mon, 23 Jul 2012 20:56:55 -0700 Subject: [PATCH 108/374] fix bug in coherence hub, respect xact_rep.ready --- uncore/uncore.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 6ea359ce..2035453c 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -367,7 +367,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH rep.bits.global_xact_id := ack_idx when (UFix(j) === init_tile_id_arr(ack_idx)) { rep.valid := send_x_rep_ack_arr.toBits.orR - sent_x_rep_ack_arr(ack_idx) := Bool(true) + sent_x_rep_ack_arr(ack_idx) := rep.ready } } } From 77364057267769be338f82a13e0e2c9a223afbb5 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Mon, 23 Jul 2012 20:56:55 -0700 Subject: [PATCH 109/374] fix bug in coherence hub, respect xact_rep.ready --- uncore/uncore.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 6ea359ce..2035453c 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -367,7 +367,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH rep.bits.global_xact_id := ack_idx when (UFix(j) === init_tile_id_arr(ack_idx)) { rep.valid := send_x_rep_ack_arr.toBits.orR - sent_x_rep_ack_arr(ack_idx) := Bool(true) + sent_x_rep_ack_arr(ack_idx) := rep.ready } } } From 1405718ca8e1a79ffc39c2727ef17a037fcef93f Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 26 Jul 2012 00:03:55 -0700 Subject: [PATCH 110/374] memory system bug fixes --- uncore/llc.scala | 4 ++-- uncore/uncore.scala | 33 ++++++++++++++++++--------------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 1c9d49b8..d95aeff7 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -378,7 +378,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da writeback.io.req(1).bits := s2.addr writeback.io.data(1).valid := io.cpu.req_data.valid writeback.io.data(1).bits := io.cpu.req_data.bits - data.io.req_data.valid := io.cpu.req_data.valid && !writeback.io.data(1).ready + data.io.req_data.valid := io.cpu.req_data.valid && writeback.io.req(1).ready memCmdArb.io.in(0) <> mshr.io.mem.req_cmd memCmdArb.io.in(1) <> writeback.io.mem.req_cmd @@ -393,7 +393,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da io.cpu.resp <> data.io.resp io.cpu.req_cmd.ready := !stall_s1 && !replay_s1 - io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready + io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready && writeback.io.req(1).ready io.mem.req_cmd <> memCmdArb.io.out io.mem.req_data <> writeback.io.mem.req_data } diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 2035453c..5c58f151 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -95,21 +95,25 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { } def doMemReqWrite(req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: PipeIO[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { - req_cmd.valid := !cmd_sent && data.valid && at_front_of_dep_queue req_cmd.bits.rw := Bool(true) - req_data.valid := data.valid && at_front_of_dep_queue req_data.bits := data.bits - lock := data.valid && at_front_of_dep_queue when(req_cmd.ready && req_cmd.valid) { cmd_sent := Bool(true) } - when(req_data.ready && at_front_of_dep_queue) { - pop_data := UFix(1) << tile_id - when (data.valid) { - mem_cnt := mem_cnt_next - when(mem_cnt_next === UFix(0)) { - pop_dep := UFix(1) << tile_id - trigger := Bool(false) + when (at_front_of_dep_queue) { + req_cmd.valid := !cmd_sent && req_data.ready + lock := Bool(true) + when (req_cmd.ready || cmd_sent) { + req_data.valid := data.valid + when(req_data.ready) { + pop_data := UFix(1) << tile_id + when (data.valid) { + mem_cnt := mem_cnt_next + when(mem_cnt === UFix(REFILL_CYCLES-1)) { + pop_dep := UFix(1) << tile_id + trigger := Bool(false) + } + } } } } @@ -141,7 +145,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) val p_req_initial_flags = Bits(width = ntiles) - p_req_initial_flags := ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id(log2Up(ntiles)-1,0))) //TODO: Broadcast only + p_req_initial_flags := (if (ntiles == 1) Bits(0) else ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id(log2Up(ntiles)-1,0)))) //TODO: Broadcast only io.busy := state != s_idle io.addr := addr_ @@ -376,16 +380,15 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests // and once we have picked a request, get the right write data - val mem_req_cmd_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemReqCmd() } + val mem_req_cmd_arb = (new Arbiter(NGLOBAL_XACTS)) { new MemReqCmd() } val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemData() } for( i <- 0 until NGLOBAL_XACTS ) { mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd - mem_req_cmd_arb.io.lock(i) <> trackerList(i).io.mem_req_lock mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock } - io.mem.req_cmd <> mem_req_cmd_arb.io.out - io.mem.req_data <> mem_req_data_arb.io.out + io.mem.req_cmd <> Queue(mem_req_cmd_arb.io.out) + io.mem.req_data <> Queue(mem_req_data_arb.io.out) // Handle probe replies, which may or may not have data for( j <- 0 until ntiles ) { From 1ae3091261f64e1f8e6de7f9e9e2afc4b3c855ce Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 26 Jul 2012 00:03:55 -0700 Subject: [PATCH 111/374] memory system bug fixes --- uncore/llc.scala | 4 ++-- uncore/uncore.scala | 33 ++++++++++++++++++--------------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 1c9d49b8..d95aeff7 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -378,7 +378,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da writeback.io.req(1).bits := s2.addr writeback.io.data(1).valid := io.cpu.req_data.valid writeback.io.data(1).bits := io.cpu.req_data.bits - data.io.req_data.valid := io.cpu.req_data.valid && !writeback.io.data(1).ready + data.io.req_data.valid := io.cpu.req_data.valid && writeback.io.req(1).ready memCmdArb.io.in(0) <> mshr.io.mem.req_cmd memCmdArb.io.in(1) <> writeback.io.mem.req_cmd @@ -393,7 +393,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da io.cpu.resp <> data.io.resp io.cpu.req_cmd.ready := !stall_s1 && !replay_s1 - io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready + io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready && writeback.io.req(1).ready io.mem.req_cmd <> memCmdArb.io.out io.mem.req_data <> writeback.io.mem.req_data } diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 2035453c..5c58f151 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -95,21 +95,25 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { } def doMemReqWrite(req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: PipeIO[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { - req_cmd.valid := !cmd_sent && data.valid && at_front_of_dep_queue req_cmd.bits.rw := Bool(true) - req_data.valid := data.valid && at_front_of_dep_queue req_data.bits := data.bits - lock := data.valid && at_front_of_dep_queue when(req_cmd.ready && req_cmd.valid) { cmd_sent := Bool(true) } - when(req_data.ready && at_front_of_dep_queue) { - pop_data := UFix(1) << tile_id - when (data.valid) { - mem_cnt := mem_cnt_next - when(mem_cnt_next === UFix(0)) { - pop_dep := UFix(1) << tile_id - trigger := Bool(false) + when (at_front_of_dep_queue) { + req_cmd.valid := !cmd_sent && req_data.ready + lock := Bool(true) + when (req_cmd.ready || cmd_sent) { + req_data.valid := data.valid + when(req_data.ready) { + pop_data := UFix(1) << tile_id + when (data.valid) { + mem_cnt := mem_cnt_next + when(mem_cnt === UFix(REFILL_CYCLES-1)) { + pop_dep := UFix(1) << tile_id + trigger := Bool(false) + } + } } } } @@ -141,7 +145,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) val p_req_initial_flags = Bits(width = ntiles) - p_req_initial_flags := ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id(log2Up(ntiles)-1,0))) //TODO: Broadcast only + p_req_initial_flags := (if (ntiles == 1) Bits(0) else ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id(log2Up(ntiles)-1,0)))) //TODO: Broadcast only io.busy := state != s_idle io.addr := addr_ @@ -376,16 +380,15 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests // and once we have picked a request, get the right write data - val mem_req_cmd_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemReqCmd() } + val mem_req_cmd_arb = (new Arbiter(NGLOBAL_XACTS)) { new MemReqCmd() } val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemData() } for( i <- 0 until NGLOBAL_XACTS ) { mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd - mem_req_cmd_arb.io.lock(i) <> trackerList(i).io.mem_req_lock mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock } - io.mem.req_cmd <> mem_req_cmd_arb.io.out - io.mem.req_data <> mem_req_data_arb.io.out + io.mem.req_cmd <> Queue(mem_req_cmd_arb.io.out) + io.mem.req_data <> Queue(mem_req_data_arb.io.out) // Handle probe replies, which may or may not have data for( j <- 0 until ntiles ) { From 0a2d284d24cda2bd6559975d6ebc823bbea1079c Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Fri, 27 Jul 2012 18:44:17 -0700 Subject: [PATCH 112/374] add reset pin to llc --- uncore/llc.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index d95aeff7..739fd791 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -305,7 +305,7 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component io.writeback_data.bits := q.io.deq.bits } -class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits]) extends Component +class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits], resetSignal: Bool = null) extends Component(resetSignal) { val io = new Bundle { val cpu = new ioMem().flip From 465f2efca71727aa952bf157f22ca27ba72d457f Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Fri, 27 Jul 2012 18:44:17 -0700 Subject: [PATCH 113/374] add reset pin to llc --- uncore/llc.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index d95aeff7..739fd791 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -305,7 +305,7 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component io.writeback_data.bits := q.io.deq.bits } -class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits]) extends Component +class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits], resetSignal: Bool = null) extends Component(resetSignal) { val io = new Bundle { val cpu = new ioMem().flip From 914b6b622d98646a51437d972f823c4ccab1a27f Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Sat, 28 Jul 2012 21:14:33 -0700 Subject: [PATCH 114/374] remove reset pin on llc --- uncore/llc.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 739fd791..d95aeff7 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -305,7 +305,7 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component io.writeback_data.bits := q.io.deq.bits } -class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits], resetSignal: Bool = null) extends Component(resetSignal) +class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits]) extends Component { val io = new Bundle { val cpu = new ioMem().flip From 4d4e28c1387b309e9247ba3da94a56b7e5da4dcf Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Sat, 28 Jul 2012 21:14:33 -0700 Subject: [PATCH 115/374] remove reset pin on llc --- uncore/llc.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 739fd791..d95aeff7 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -305,7 +305,7 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component io.writeback_data.bits := q.io.deq.bits } -class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits], resetSignal: Bool = null) extends Component(resetSignal) +class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits]) extends Component { val io = new Bundle { val cpu = new ioMem().flip From 85dc34df803dae3d715531a2a85a8c5793acbc62 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 30 Jul 2012 20:12:11 -0700 Subject: [PATCH 116/374] further pipeline the LLC --- uncore/llc.scala | 84 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 62 insertions(+), 22 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index d95aeff7..89fc5e2f 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -14,7 +14,6 @@ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) ex val wmask = gen.asInput val rdata = gen.asOutput } - require(readLatency >= 0 && readLatency <= 2) val data = gen val colMux = if (2*data.width <= leaf.data.width && n > leaf.n) 1 << math.floor(math.log(leaf.data.width/data.width)/math.log(2)).toInt else 1 val nWide = if (data.width > leaf.data.width) 1+(data.width-1)/leaf.data.width else 1 @@ -28,9 +27,6 @@ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) ex val cond = Vec(nDeep) { Bool() } val ren = Vec(nDeep) { Bool() } val reg_ren = Vec(nDeep) { Reg() { Bool() } } - val reg2_ren = Vec(nDeep) { Reg() { Bool() } } - val reg_raddr = Vec(nDeep) { Reg() { UFix() } } - val reg2_raddr = Vec(nDeep) { Reg() { UFix() } } val renOut = Vec(nDeep) { Bool() } val raddrOut = Vec(nDeep) { UFix() } val rdata = Vec(nDeep) { Vec(nWide) { Bits() } } @@ -40,11 +36,14 @@ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) ex cond(i) := (if (nDeep == 1) io.en else io.en && UFix(i) === io.addr(log2Up(n)-1, log2Up(n/nDeep))) ren(i) := cond(i) && !io.rw reg_ren(i) := ren(i) - reg2_ren(i) := reg_ren(i) - when (ren(i)) { reg_raddr(i) := io.addr } - when (reg_ren(i)) { reg2_raddr(i) := reg_raddr(i) } - renOut(i) := (if (readLatency > 1) reg2_ren(i) else if (readLatency > 0) reg_ren(i) else ren(i)) - raddrOut(i) := (if (readLatency > 1) reg2_raddr(i) else if (readLatency > 0) reg_raddr(i) else io.addr) + + renOut(i) := ren(i) + raddrOut(i) := io.addr + if (readLatency > 0) { + val r = Pipe(ren(i), io.addr, readLatency) + renOut(i) := r.valid + raddrOut(i) := r.bits + } for (j <- 0 until nWide) { val mem = leaf.clone @@ -64,11 +63,8 @@ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) ex dout = mem(idx) } else if (readLatency == 1) { dout = dout1 - } else { - val dout2 = Reg() { Bits() } - when (reg_ren(i)) { dout2 := dout1 } - dout = dout2 - } + } else + dout = Pipe(reg_ren(i), dout1, readLatency-1).bits rdata(i)(j) := dout } @@ -238,7 +234,7 @@ class LLCWriteback(requestors: Int) extends Component io.mem.req_data.bits := io.data(who).bits } -class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component +class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Component { val io = new Bundle { val req = (new FIFOIO) { new LLCDataReq(ways) }.flip @@ -251,13 +247,13 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component val mem_resp_way = UFix(INPUT, log2Up(ways)) } - val data = new BigMem(sets*ways*REFILL_CYCLES, 2, leaf)(Bits(width = MEM_DATA_BITS)) + val data = new BigMem(sets*ways*REFILL_CYCLES, latency, leaf)(Bits(width = MEM_DATA_BITS)) class QEntry extends MemResp { val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] } - val q = (new queue(4)) { new QEntry } - val qReady = q.io.count <= UFix(q.entries - 3) + val q = (new queue(latency+2)) { new QEntry } + val qReady = q.io.count <= UFix(q.entries-latency-1) val valid = Reg(resetVal = Bool(false)) val req = Reg() { io.req.bits.clone } val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) @@ -287,10 +283,11 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component data.io.wdata := io.mem_resp.bits.data } - q.io.enq.valid := Reg(Reg(data.io.en && !data.io.rw, resetVal = Bool(false)), resetVal = Bool(false)) - q.io.enq.bits.tag := Reg(Reg(Mux(valid, req.tag, io.req.bits.tag))) + val tagPipe = Pipe(data.io.en && !data.io.rw, Mux(valid, req.tag, io.req.bits.tag), latency) + q.io.enq.valid := tagPipe.valid + q.io.enq.bits.tag := tagPipe.bits + q.io.enq.bits.isWriteback := Pipe(Mux(valid, req.isWriteback, io.req.bits.isWriteback), Bool(false), latency).valid q.io.enq.bits.data := data.io.rdata - q.io.enq.bits.isWriteback := Reg(Reg(Mux(valid, req.isWriteback, io.req.bits.isWriteback))) io.req.ready := !valid && Mux(io.req.bits.isWriteback, io.writeback.ready, Bool(true)) io.req_data.ready := !io.mem_resp.valid && Mux(valid, req.rw, io.req.valid && io.req.bits.rw) @@ -305,6 +302,49 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component io.writeback_data.bits := q.io.deq.bits } +class MemReqArb(n: Int) extends Component // UNTESTED +{ + val io = new Bundle { + val cpu = Vec(n) { new ioMem().flip } + val mem = new ioMem + } + + val lock = Reg(resetVal = Bool(false)) + val locker = Reg() { UFix() } + + val arb = new RRArbiter(n)(new MemReqCmd) + val respWho = io.mem.resp.bits.tag(log2Up(n)-1,0) + val respTag = io.mem.resp.bits.tag >> UFix(log2Up(n)) + for (i <- 0 until n) { + val me = UFix(i, log2Up(n)) + arb.io.in(i).valid := io.cpu(i).req_cmd.valid + arb.io.in(i).bits := io.cpu(i).req_cmd.bits + arb.io.in(i).bits.tag := Cat(io.cpu(i).req_cmd.bits.tag, me) + io.cpu(i).req_cmd.ready := arb.io.in(i).ready + io.cpu(i).req_data.ready := Bool(false) + + val getLock = io.cpu(i).req_cmd.fire() && io.cpu(i).req_cmd.bits.rw && !lock + val haveLock = lock && locker === me + when (getLock) { + lock := Bool(true) + locker := UFix(i) + } + when (getLock || haveLock) { + io.cpu(i).req_data.ready := io.mem.req_data.ready + io.mem.req_data.valid := Bool(true) + io.mem.req_data.bits := io.cpu(i).req_data.bits + } + + io.cpu(i).resp.valid := io.mem.resp.valid && respWho === me + io.cpu(i).resp.bits := io.mem.resp.bits + io.cpu(i).resp.bits.tag := respTag + } + io.mem.resp.ready := io.cpu(respWho).resp.ready + + val unlock = Counter(io.mem.req_data.fire(), REFILL_CYCLES)._2 + when (unlock) { lock := Bool(false) } +} + class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits]) extends Component { val io = new Bundle { @@ -319,7 +359,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val dataArb = (new Arbiter(2)) { new LLCDataReq(ways) } val mshr = new LLCMSHRFile(sets, ways, outstanding) val tags = new BigMem(sets, 1, tagLeaf)(Bits(width = metaWidth*ways)) - val data = new LLCData(sets, ways, dataLeaf) + val data = new LLCData(3, sets, ways, dataLeaf) val writeback = new LLCWriteback(2) val initCount = Reg(resetVal = UFix(0, log2Up(sets+1))) From 8db233c9b721abf66a47caf2d79c4e5c97c9a8fd Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 30 Jul 2012 20:12:11 -0700 Subject: [PATCH 117/374] further pipeline the LLC --- uncore/llc.scala | 84 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 62 insertions(+), 22 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index d95aeff7..89fc5e2f 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -14,7 +14,6 @@ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) ex val wmask = gen.asInput val rdata = gen.asOutput } - require(readLatency >= 0 && readLatency <= 2) val data = gen val colMux = if (2*data.width <= leaf.data.width && n > leaf.n) 1 << math.floor(math.log(leaf.data.width/data.width)/math.log(2)).toInt else 1 val nWide = if (data.width > leaf.data.width) 1+(data.width-1)/leaf.data.width else 1 @@ -28,9 +27,6 @@ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) ex val cond = Vec(nDeep) { Bool() } val ren = Vec(nDeep) { Bool() } val reg_ren = Vec(nDeep) { Reg() { Bool() } } - val reg2_ren = Vec(nDeep) { Reg() { Bool() } } - val reg_raddr = Vec(nDeep) { Reg() { UFix() } } - val reg2_raddr = Vec(nDeep) { Reg() { UFix() } } val renOut = Vec(nDeep) { Bool() } val raddrOut = Vec(nDeep) { UFix() } val rdata = Vec(nDeep) { Vec(nWide) { Bits() } } @@ -40,11 +36,14 @@ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) ex cond(i) := (if (nDeep == 1) io.en else io.en && UFix(i) === io.addr(log2Up(n)-1, log2Up(n/nDeep))) ren(i) := cond(i) && !io.rw reg_ren(i) := ren(i) - reg2_ren(i) := reg_ren(i) - when (ren(i)) { reg_raddr(i) := io.addr } - when (reg_ren(i)) { reg2_raddr(i) := reg_raddr(i) } - renOut(i) := (if (readLatency > 1) reg2_ren(i) else if (readLatency > 0) reg_ren(i) else ren(i)) - raddrOut(i) := (if (readLatency > 1) reg2_raddr(i) else if (readLatency > 0) reg_raddr(i) else io.addr) + + renOut(i) := ren(i) + raddrOut(i) := io.addr + if (readLatency > 0) { + val r = Pipe(ren(i), io.addr, readLatency) + renOut(i) := r.valid + raddrOut(i) := r.bits + } for (j <- 0 until nWide) { val mem = leaf.clone @@ -64,11 +63,8 @@ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) ex dout = mem(idx) } else if (readLatency == 1) { dout = dout1 - } else { - val dout2 = Reg() { Bits() } - when (reg_ren(i)) { dout2 := dout1 } - dout = dout2 - } + } else + dout = Pipe(reg_ren(i), dout1, readLatency-1).bits rdata(i)(j) := dout } @@ -238,7 +234,7 @@ class LLCWriteback(requestors: Int) extends Component io.mem.req_data.bits := io.data(who).bits } -class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component +class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Component { val io = new Bundle { val req = (new FIFOIO) { new LLCDataReq(ways) }.flip @@ -251,13 +247,13 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component val mem_resp_way = UFix(INPUT, log2Up(ways)) } - val data = new BigMem(sets*ways*REFILL_CYCLES, 2, leaf)(Bits(width = MEM_DATA_BITS)) + val data = new BigMem(sets*ways*REFILL_CYCLES, latency, leaf)(Bits(width = MEM_DATA_BITS)) class QEntry extends MemResp { val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] } - val q = (new queue(4)) { new QEntry } - val qReady = q.io.count <= UFix(q.entries - 3) + val q = (new queue(latency+2)) { new QEntry } + val qReady = q.io.count <= UFix(q.entries-latency-1) val valid = Reg(resetVal = Bool(false)) val req = Reg() { io.req.bits.clone } val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) @@ -287,10 +283,11 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component data.io.wdata := io.mem_resp.bits.data } - q.io.enq.valid := Reg(Reg(data.io.en && !data.io.rw, resetVal = Bool(false)), resetVal = Bool(false)) - q.io.enq.bits.tag := Reg(Reg(Mux(valid, req.tag, io.req.bits.tag))) + val tagPipe = Pipe(data.io.en && !data.io.rw, Mux(valid, req.tag, io.req.bits.tag), latency) + q.io.enq.valid := tagPipe.valid + q.io.enq.bits.tag := tagPipe.bits + q.io.enq.bits.isWriteback := Pipe(Mux(valid, req.isWriteback, io.req.bits.isWriteback), Bool(false), latency).valid q.io.enq.bits.data := data.io.rdata - q.io.enq.bits.isWriteback := Reg(Reg(Mux(valid, req.isWriteback, io.req.bits.isWriteback))) io.req.ready := !valid && Mux(io.req.bits.isWriteback, io.writeback.ready, Bool(true)) io.req_data.ready := !io.mem_resp.valid && Mux(valid, req.rw, io.req.valid && io.req.bits.rw) @@ -305,6 +302,49 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component io.writeback_data.bits := q.io.deq.bits } +class MemReqArb(n: Int) extends Component // UNTESTED +{ + val io = new Bundle { + val cpu = Vec(n) { new ioMem().flip } + val mem = new ioMem + } + + val lock = Reg(resetVal = Bool(false)) + val locker = Reg() { UFix() } + + val arb = new RRArbiter(n)(new MemReqCmd) + val respWho = io.mem.resp.bits.tag(log2Up(n)-1,0) + val respTag = io.mem.resp.bits.tag >> UFix(log2Up(n)) + for (i <- 0 until n) { + val me = UFix(i, log2Up(n)) + arb.io.in(i).valid := io.cpu(i).req_cmd.valid + arb.io.in(i).bits := io.cpu(i).req_cmd.bits + arb.io.in(i).bits.tag := Cat(io.cpu(i).req_cmd.bits.tag, me) + io.cpu(i).req_cmd.ready := arb.io.in(i).ready + io.cpu(i).req_data.ready := Bool(false) + + val getLock = io.cpu(i).req_cmd.fire() && io.cpu(i).req_cmd.bits.rw && !lock + val haveLock = lock && locker === me + when (getLock) { + lock := Bool(true) + locker := UFix(i) + } + when (getLock || haveLock) { + io.cpu(i).req_data.ready := io.mem.req_data.ready + io.mem.req_data.valid := Bool(true) + io.mem.req_data.bits := io.cpu(i).req_data.bits + } + + io.cpu(i).resp.valid := io.mem.resp.valid && respWho === me + io.cpu(i).resp.bits := io.mem.resp.bits + io.cpu(i).resp.bits.tag := respTag + } + io.mem.resp.ready := io.cpu(respWho).resp.ready + + val unlock = Counter(io.mem.req_data.fire(), REFILL_CYCLES)._2 + when (unlock) { lock := Bool(false) } +} + class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits]) extends Component { val io = new Bundle { @@ -319,7 +359,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val dataArb = (new Arbiter(2)) { new LLCDataReq(ways) } val mshr = new LLCMSHRFile(sets, ways, outstanding) val tags = new BigMem(sets, 1, tagLeaf)(Bits(width = metaWidth*ways)) - val data = new LLCData(sets, ways, dataLeaf) + val data = new LLCData(3, sets, ways, dataLeaf) val writeback = new LLCWriteback(2) val initCount = Reg(resetVal = UFix(0, log2Up(sets+1))) From 7b9cfd0b90b7f76a5bda02393af873a8e9e9868b Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 31 Jul 2012 17:44:53 -0700 Subject: [PATCH 118/374] pipeline LLC further --- uncore/llc.scala | 104 ++++++++++++++++++++++------------------------- 1 file changed, 48 insertions(+), 56 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 89fc5e2f..546da774 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -4,14 +4,17 @@ import Chisel._ import Node._ import Constants._ -class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component +class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component { - val io = new Bundle { + class Inputs extends Bundle { val addr = UFix(INPUT, log2Up(n)) - val en = Bool(INPUT) val rw = Bool(INPUT) val wdata = gen.asInput val wmask = gen.asInput + override def clone = new Inputs().asInstanceOf[this.type] + } + val io = new Bundle { + val in = new PipeIO()(new Inputs).flip val rdata = gen.asOutput } val data = gen @@ -21,63 +24,52 @@ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) ex if (nDeep > 1 || colMux > 1) require(isPow2(n) && isPow2(leaf.n)) - val idx = io.addr(log2Up(n/nDeep/colMux)-1, 0) val rdataDeep = Vec(nDeep) { Bits() } val rdataSel = Vec(nDeep) { Bool() } - val cond = Vec(nDeep) { Bool() } - val ren = Vec(nDeep) { Bool() } - val reg_ren = Vec(nDeep) { Reg() { Bool() } } - val renOut = Vec(nDeep) { Bool() } - val raddrOut = Vec(nDeep) { UFix() } - val rdata = Vec(nDeep) { Vec(nWide) { Bits() } } - val wdata = io.wdata.toBits - val wmask = io.wmask.toBits for (i <- 0 until nDeep) { - cond(i) := (if (nDeep == 1) io.en else io.en && UFix(i) === io.addr(log2Up(n)-1, log2Up(n/nDeep))) - ren(i) := cond(i) && !io.rw - reg_ren(i) := ren(i) + val in = Pipe(io.in.valid && (if (nDeep == 1) Bool(true) else UFix(i) === io.in.bits.addr(log2Up(n)-1, log2Up(n/nDeep))), io.in.bits, preLatency) + val idx = in.bits.addr(log2Up(n/nDeep/colMux)-1, 0) + val wdata = in.bits.wdata.toBits + val wmask = in.bits.wmask.toBits + val ren = in.valid && !in.bits.rw + val reg_ren = Reg(ren) + val rdata = Vec(nWide) { Bits() } - renOut(i) := ren(i) - raddrOut(i) := io.addr - if (readLatency > 0) { - val r = Pipe(ren(i), io.addr, readLatency) - renOut(i) := r.valid - raddrOut(i) := r.bits - } + val r = Pipe(ren, in.bits.addr, postLatency) for (j <- 0 until nWide) { val mem = leaf.clone var dout: Bits = null - val dout1 = if (readLatency > 0) Reg() { Bits() } else null + val dout1 = if (postLatency > 0) Reg() { Bits() } else null var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) if (colMux > 1) - wmask0 = wmask0 & FillInterleaved(gen.width, UFixToOH(io.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) + wmask0 = wmask0 & FillInterleaved(gen.width, UFixToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) - when (cond(i)) { - when (io.rw) { mem.write(idx, wdata0, wmask0) } - .otherwise { if (readLatency > 0) dout1 := mem(idx) } + when (in.valid) { + when (in.bits.rw) { mem.write(idx, wdata0, wmask0) } + .otherwise { if (postLatency > 0) dout1 := mem(idx) } } - if (readLatency == 0) { + if (postLatency == 0) { dout = mem(idx) - } else if (readLatency == 1) { + } else if (postLatency == 1) { dout = dout1 } else - dout = Pipe(reg_ren(i), dout1, readLatency-1).bits + dout = Pipe(reg_ren, dout1, postLatency-1).bits - rdata(i)(j) := dout + rdata(j) := dout } - val rdataWide = rdata(i).reduceLeft((x, y) => Cat(y, x)) + val rdataWide = rdata.reduceLeft((x, y) => Cat(y, x)) var colMuxOut = rdataWide if (colMux > 1) { val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.width*(k+1)-1, gen.width*k))) { Bits() } - colMuxOut = colMuxIn(raddrOut(i)(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) + colMuxOut = colMuxIn(r.bits(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) } rdataDeep(i) := colMuxOut - rdataSel(i) := renOut(i) + rdataSel(i) := r.valid } io.rdata := Mux1H(rdataSel, rdataDeep) @@ -247,7 +239,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Compo val mem_resp_way = UFix(INPUT, log2Up(ways)) } - val data = new BigMem(sets*ways*REFILL_CYCLES, latency, leaf)(Bits(width = MEM_DATA_BITS)) + val data = new BigMem(sets*ways*REFILL_CYCLES, 1, latency-1, leaf)(Bits(width = MEM_DATA_BITS)) class QEntry extends MemResp { val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] @@ -259,31 +251,31 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Compo val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) val refillCount = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - when (data.io.en && !io.mem_resp.valid) { + when (data.io.in.valid && !io.mem_resp.valid) { count := count + UFix(1) when (valid && count === UFix(REFILL_CYCLES-1)) { valid := Bool(false) } } when (io.req.valid && io.req.ready) { valid := Bool(true); req := io.req.bits } when (io.mem_resp.valid) { refillCount := refillCount + UFix(1) } - data.io.en := io.req.valid && io.req.ready && Mux(io.req.bits.rw, io.req_data.valid, qReady) - data.io.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUFix - data.io.rw := io.req.bits.rw - data.io.wdata := io.req_data.bits.data - data.io.wmask := Fix(-1, io.req_data.bits.data.width) + data.io.in.valid := io.req.valid && io.req.ready && Mux(io.req.bits.rw, io.req_data.valid, qReady) + data.io.in.bits.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUFix + data.io.in.bits.rw := io.req.bits.rw + data.io.in.bits.wdata := io.req_data.bits.data + data.io.in.bits.wmask := Fix(-1, io.req_data.bits.data.width) when (valid) { - data.io.en := Mux(req.rw, io.req_data.valid, qReady) - data.io.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUFix - data.io.rw := req.rw + data.io.in.valid := Mux(req.rw, io.req_data.valid, qReady) + data.io.in.bits.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUFix + data.io.in.bits.rw := req.rw } when (io.mem_resp.valid) { - data.io.en := Bool(true) - data.io.addr := Cat(io.mem_resp_way, io.mem_resp_set, refillCount).toUFix - data.io.rw := Bool(true) - data.io.wdata := io.mem_resp.bits.data + data.io.in.valid := Bool(true) + data.io.in.bits.addr := Cat(io.mem_resp_way, io.mem_resp_set, refillCount).toUFix + data.io.in.bits.rw := Bool(true) + data.io.in.bits.wdata := io.mem_resp.bits.data } - val tagPipe = Pipe(data.io.en && !data.io.rw, Mux(valid, req.tag, io.req.bits.tag), latency) + val tagPipe = Pipe(data.io.in.valid && !data.io.in.bits.rw, Mux(valid, req.tag, io.req.bits.tag), latency) q.io.enq.valid := tagPipe.valid q.io.enq.bits.tag := tagPipe.bits q.io.enq.bits.isWriteback := Pipe(Mux(valid, req.isWriteback, io.req.bits.isWriteback), Bool(false), latency).valid @@ -358,8 +350,8 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val memCmdArb = (new Arbiter(2)) { new MemReqCmd } val dataArb = (new Arbiter(2)) { new LLCDataReq(ways) } val mshr = new LLCMSHRFile(sets, ways, outstanding) - val tags = new BigMem(sets, 1, tagLeaf)(Bits(width = metaWidth*ways)) - val data = new LLCData(3, sets, ways, dataLeaf) + val tags = new BigMem(sets, 0, 1, tagLeaf)(Bits(width = metaWidth*ways)) + val data = new LLCData(4, sets, ways, dataLeaf) val writeback = new LLCWriteback(2) val initCount = Reg(resetVal = UFix(0, log2Up(sets+1))) @@ -392,11 +384,11 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val setDirty = s2_valid && s2.rw && s2_hit && !s2_hit_dirty stall_s1 := initialize || mshr.io.tag.valid || setDirty || s2_valid && !s2_hit || stall_s2 - tags.io.en := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || setDirty || mshr.io.tag.valid - tags.io.addr := Mux(initialize, initCount, Mux(setDirty, s2.addr, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)))(log2Up(sets)-1,0)) - tags.io.rw := initialize || setDirty || mshr.io.tag.valid - tags.io.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) - tags.io.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way)))) + tags.io.in.valid := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || setDirty || mshr.io.tag.valid + tags.io.in.bits.addr := Mux(initialize, initCount, Mux(setDirty, s2.addr, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)))(log2Up(sets)-1,0)) + tags.io.in.bits.rw := initialize || setDirty || mshr.io.tag.valid + tags.io.in.bits.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) + tags.io.in.bits.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way)))) mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw mshr.io.cpu.bits := s2 From 7a75334bb97d103f7c4ff1c3b936434c5683bebf Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 31 Jul 2012 17:44:53 -0700 Subject: [PATCH 119/374] pipeline LLC further --- uncore/llc.scala | 104 ++++++++++++++++++++++------------------------- 1 file changed, 48 insertions(+), 56 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 89fc5e2f..546da774 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -4,14 +4,17 @@ import Chisel._ import Node._ import Constants._ -class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component +class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component { - val io = new Bundle { + class Inputs extends Bundle { val addr = UFix(INPUT, log2Up(n)) - val en = Bool(INPUT) val rw = Bool(INPUT) val wdata = gen.asInput val wmask = gen.asInput + override def clone = new Inputs().asInstanceOf[this.type] + } + val io = new Bundle { + val in = new PipeIO()(new Inputs).flip val rdata = gen.asOutput } val data = gen @@ -21,63 +24,52 @@ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) ex if (nDeep > 1 || colMux > 1) require(isPow2(n) && isPow2(leaf.n)) - val idx = io.addr(log2Up(n/nDeep/colMux)-1, 0) val rdataDeep = Vec(nDeep) { Bits() } val rdataSel = Vec(nDeep) { Bool() } - val cond = Vec(nDeep) { Bool() } - val ren = Vec(nDeep) { Bool() } - val reg_ren = Vec(nDeep) { Reg() { Bool() } } - val renOut = Vec(nDeep) { Bool() } - val raddrOut = Vec(nDeep) { UFix() } - val rdata = Vec(nDeep) { Vec(nWide) { Bits() } } - val wdata = io.wdata.toBits - val wmask = io.wmask.toBits for (i <- 0 until nDeep) { - cond(i) := (if (nDeep == 1) io.en else io.en && UFix(i) === io.addr(log2Up(n)-1, log2Up(n/nDeep))) - ren(i) := cond(i) && !io.rw - reg_ren(i) := ren(i) + val in = Pipe(io.in.valid && (if (nDeep == 1) Bool(true) else UFix(i) === io.in.bits.addr(log2Up(n)-1, log2Up(n/nDeep))), io.in.bits, preLatency) + val idx = in.bits.addr(log2Up(n/nDeep/colMux)-1, 0) + val wdata = in.bits.wdata.toBits + val wmask = in.bits.wmask.toBits + val ren = in.valid && !in.bits.rw + val reg_ren = Reg(ren) + val rdata = Vec(nWide) { Bits() } - renOut(i) := ren(i) - raddrOut(i) := io.addr - if (readLatency > 0) { - val r = Pipe(ren(i), io.addr, readLatency) - renOut(i) := r.valid - raddrOut(i) := r.bits - } + val r = Pipe(ren, in.bits.addr, postLatency) for (j <- 0 until nWide) { val mem = leaf.clone var dout: Bits = null - val dout1 = if (readLatency > 0) Reg() { Bits() } else null + val dout1 = if (postLatency > 0) Reg() { Bits() } else null var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) if (colMux > 1) - wmask0 = wmask0 & FillInterleaved(gen.width, UFixToOH(io.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) + wmask0 = wmask0 & FillInterleaved(gen.width, UFixToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) - when (cond(i)) { - when (io.rw) { mem.write(idx, wdata0, wmask0) } - .otherwise { if (readLatency > 0) dout1 := mem(idx) } + when (in.valid) { + when (in.bits.rw) { mem.write(idx, wdata0, wmask0) } + .otherwise { if (postLatency > 0) dout1 := mem(idx) } } - if (readLatency == 0) { + if (postLatency == 0) { dout = mem(idx) - } else if (readLatency == 1) { + } else if (postLatency == 1) { dout = dout1 } else - dout = Pipe(reg_ren(i), dout1, readLatency-1).bits + dout = Pipe(reg_ren, dout1, postLatency-1).bits - rdata(i)(j) := dout + rdata(j) := dout } - val rdataWide = rdata(i).reduceLeft((x, y) => Cat(y, x)) + val rdataWide = rdata.reduceLeft((x, y) => Cat(y, x)) var colMuxOut = rdataWide if (colMux > 1) { val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.width*(k+1)-1, gen.width*k))) { Bits() } - colMuxOut = colMuxIn(raddrOut(i)(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) + colMuxOut = colMuxIn(r.bits(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) } rdataDeep(i) := colMuxOut - rdataSel(i) := renOut(i) + rdataSel(i) := r.valid } io.rdata := Mux1H(rdataSel, rdataDeep) @@ -247,7 +239,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Compo val mem_resp_way = UFix(INPUT, log2Up(ways)) } - val data = new BigMem(sets*ways*REFILL_CYCLES, latency, leaf)(Bits(width = MEM_DATA_BITS)) + val data = new BigMem(sets*ways*REFILL_CYCLES, 1, latency-1, leaf)(Bits(width = MEM_DATA_BITS)) class QEntry extends MemResp { val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] @@ -259,31 +251,31 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Compo val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) val refillCount = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - when (data.io.en && !io.mem_resp.valid) { + when (data.io.in.valid && !io.mem_resp.valid) { count := count + UFix(1) when (valid && count === UFix(REFILL_CYCLES-1)) { valid := Bool(false) } } when (io.req.valid && io.req.ready) { valid := Bool(true); req := io.req.bits } when (io.mem_resp.valid) { refillCount := refillCount + UFix(1) } - data.io.en := io.req.valid && io.req.ready && Mux(io.req.bits.rw, io.req_data.valid, qReady) - data.io.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUFix - data.io.rw := io.req.bits.rw - data.io.wdata := io.req_data.bits.data - data.io.wmask := Fix(-1, io.req_data.bits.data.width) + data.io.in.valid := io.req.valid && io.req.ready && Mux(io.req.bits.rw, io.req_data.valid, qReady) + data.io.in.bits.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUFix + data.io.in.bits.rw := io.req.bits.rw + data.io.in.bits.wdata := io.req_data.bits.data + data.io.in.bits.wmask := Fix(-1, io.req_data.bits.data.width) when (valid) { - data.io.en := Mux(req.rw, io.req_data.valid, qReady) - data.io.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUFix - data.io.rw := req.rw + data.io.in.valid := Mux(req.rw, io.req_data.valid, qReady) + data.io.in.bits.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUFix + data.io.in.bits.rw := req.rw } when (io.mem_resp.valid) { - data.io.en := Bool(true) - data.io.addr := Cat(io.mem_resp_way, io.mem_resp_set, refillCount).toUFix - data.io.rw := Bool(true) - data.io.wdata := io.mem_resp.bits.data + data.io.in.valid := Bool(true) + data.io.in.bits.addr := Cat(io.mem_resp_way, io.mem_resp_set, refillCount).toUFix + data.io.in.bits.rw := Bool(true) + data.io.in.bits.wdata := io.mem_resp.bits.data } - val tagPipe = Pipe(data.io.en && !data.io.rw, Mux(valid, req.tag, io.req.bits.tag), latency) + val tagPipe = Pipe(data.io.in.valid && !data.io.in.bits.rw, Mux(valid, req.tag, io.req.bits.tag), latency) q.io.enq.valid := tagPipe.valid q.io.enq.bits.tag := tagPipe.bits q.io.enq.bits.isWriteback := Pipe(Mux(valid, req.isWriteback, io.req.bits.isWriteback), Bool(false), latency).valid @@ -358,8 +350,8 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val memCmdArb = (new Arbiter(2)) { new MemReqCmd } val dataArb = (new Arbiter(2)) { new LLCDataReq(ways) } val mshr = new LLCMSHRFile(sets, ways, outstanding) - val tags = new BigMem(sets, 1, tagLeaf)(Bits(width = metaWidth*ways)) - val data = new LLCData(3, sets, ways, dataLeaf) + val tags = new BigMem(sets, 0, 1, tagLeaf)(Bits(width = metaWidth*ways)) + val data = new LLCData(4, sets, ways, dataLeaf) val writeback = new LLCWriteback(2) val initCount = Reg(resetVal = UFix(0, log2Up(sets+1))) @@ -392,11 +384,11 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val setDirty = s2_valid && s2.rw && s2_hit && !s2_hit_dirty stall_s1 := initialize || mshr.io.tag.valid || setDirty || s2_valid && !s2_hit || stall_s2 - tags.io.en := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || setDirty || mshr.io.tag.valid - tags.io.addr := Mux(initialize, initCount, Mux(setDirty, s2.addr, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)))(log2Up(sets)-1,0)) - tags.io.rw := initialize || setDirty || mshr.io.tag.valid - tags.io.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) - tags.io.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way)))) + tags.io.in.valid := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || setDirty || mshr.io.tag.valid + tags.io.in.bits.addr := Mux(initialize, initCount, Mux(setDirty, s2.addr, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)))(log2Up(sets)-1,0)) + tags.io.in.bits.rw := initialize || setDirty || mshr.io.tag.valid + tags.io.in.bits.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) + tags.io.in.bits.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way)))) mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw mshr.io.cpu.bits := s2 From 92b7504c9a02c4bd1c067bc71eb1223fd212c05e Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 3 Aug 2012 18:59:37 -0700 Subject: [PATCH 120/374] fix control bug in LLC structural hazard on tag ram caused deadlock --- uncore/llc.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 546da774..eece03ed 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -154,13 +154,14 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val conflicts = Cat(Bits(0), (0 until outstanding).map(i => valid(i) && io.cpu.bits.addr(log2Up(sets)-1, 0) === mshr(i).addr(log2Up(sets)-1, 0)):_*) io.cpu.ready := !conflicts.orR && !validBits.andR - io.data.valid := replay && io.tag.ready || writeback + io.data.valid := writeback io.data.bits.rw := Bool(false) io.data.bits.tag := mshr(replayId).tag io.data.bits.isWriteback := Bool(true) io.data.bits.addr := Cat(mshr(writebackId).old_tag, mshr(writebackId).addr(log2Up(sets)-1, 0)).toUFix io.data.bits.way := mshr(writebackId).way when (replay) { + io.data.valid := io.tag.ready io.data.bits.isWriteback := Bool(false) io.data.bits.addr := mshr(replayId).addr io.data.bits.way := mshr(replayId).way From e346f21725e3de2ac2939db11e644f29e9e3ac3b Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 3 Aug 2012 18:59:37 -0700 Subject: [PATCH 121/374] fix control bug in LLC structural hazard on tag ram caused deadlock --- uncore/llc.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 546da774..eece03ed 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -154,13 +154,14 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val conflicts = Cat(Bits(0), (0 until outstanding).map(i => valid(i) && io.cpu.bits.addr(log2Up(sets)-1, 0) === mshr(i).addr(log2Up(sets)-1, 0)):_*) io.cpu.ready := !conflicts.orR && !validBits.andR - io.data.valid := replay && io.tag.ready || writeback + io.data.valid := writeback io.data.bits.rw := Bool(false) io.data.bits.tag := mshr(replayId).tag io.data.bits.isWriteback := Bool(true) io.data.bits.addr := Cat(mshr(writebackId).old_tag, mshr(writebackId).addr(log2Up(sets)-1, 0)).toUFix io.data.bits.way := mshr(writebackId).way when (replay) { + io.data.valid := io.tag.ready io.data.bits.isWriteback := Bool(false) io.data.bits.addr := mshr(replayId).addr io.data.bits.way := mshr(replayId).way From 962423d2d150944c6e37e5ccf0b08aac2a05e55f Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 3 Aug 2012 19:00:03 -0700 Subject: [PATCH 122/374] fix deadlock in coherence hub --- uncore/uncore.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 5c58f151..a1726242 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -101,8 +101,8 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { cmd_sent := Bool(true) } when (at_front_of_dep_queue) { - req_cmd.valid := !cmd_sent && req_data.ready - lock := Bool(true) + req_cmd.valid := !cmd_sent && req_data.ready && data.valid + lock := data.valid || cmd_sent when (req_cmd.ready || cmd_sent) { req_data.valid := data.valid when(req_data.ready) { From 875f3622af6520e2985312b07aa0d2ca65732403 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 3 Aug 2012 19:00:03 -0700 Subject: [PATCH 123/374] fix deadlock in coherence hub --- uncore/uncore.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 5c58f151..a1726242 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -101,8 +101,8 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { cmd_sent := Bool(true) } when (at_front_of_dep_queue) { - req_cmd.valid := !cmd_sent && req_data.ready - lock := Bool(true) + req_cmd.valid := !cmd_sent && req_data.ready && data.valid + lock := data.valid || cmd_sent when (req_cmd.ready || cmd_sent) { req_data.valid := data.valid when(req_data.ready) { From 115c25c34b2f5fe0524b6108e8088c912d5cb428 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 6 Aug 2012 17:10:04 -0700 Subject: [PATCH 124/374] fix some LLC control bugs --- uncore/llc.scala | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index eece03ed..64d48ce8 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -383,15 +383,20 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val repl_way = LFSR16(s2_valid)(log2Up(ways)-1, 0) val repl_tag = s2_tags(repl_way).toUFix val setDirty = s2_valid && s2.rw && s2_hit && !s2_hit_dirty - stall_s1 := initialize || mshr.io.tag.valid || setDirty || s2_valid && !s2_hit || stall_s2 + stall_s1 := initialize || stall_s2 - tags.io.in.valid := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || setDirty || mshr.io.tag.valid - tags.io.in.bits.addr := Mux(initialize, initCount, Mux(setDirty, s2.addr, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)))(log2Up(sets)-1,0)) - tags.io.in.bits.rw := initialize || setDirty || mshr.io.tag.valid - tags.io.in.bits.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) - tags.io.in.bits.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way)))) + val tag_we = setDirty || mshr.io.tag.valid + val tag_waddr = Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(log2Up(sets)-1,0) + val tag_wdata = Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)) + val tag_wway = Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way) + tags.io.in.valid := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || tag_we + tags.io.in.bits.addr := Mux(initialize, initCount, Mux(tag_we, tag_waddr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)(log2Up(sets)-1,0))) + tags.io.in.bits.rw := initialize || tag_we + tags.io.in.bits.wdata := Mux(initialize, UFix(0), Fill(ways, tag_wdata)) + tags.io.in.bits.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(tag_wway))) + when (tag_we && Mux(stall_s2, s2.addr, s1.addr)(log2Up(sets)-1,0) === tag_waddr) { s2_tags(tag_wway) := tag_wdata } - mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw + mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw && dataArb.io.in(1).ready && writeback.io.req(0).ready // stall_s2 mshr.io.cpu.bits := s2 mshr.io.repl_way := repl_way mshr.io.repl_dirty := repl_tag(tagWidth+1, tagWidth).andR @@ -404,29 +409,29 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da data.io.mem_resp_set := mshr.io.mem_resp_set data.io.mem_resp_way := mshr.io.mem_resp_way data.io.req_data.bits := io.cpu.req_data.bits + data.io.req_data.valid := io.cpu.req_data.valid writeback.io.req(0) <> data.io.writeback writeback.io.data(0) <> data.io.writeback_data - writeback.io.req(1).valid := s2_valid && !s2_hit && s2.rw + writeback.io.req(1).valid := s2_valid && !s2_hit && s2.rw && dataArb.io.in(1).ready && mshr.io.cpu.ready // stall_s2 writeback.io.req(1).bits := s2.addr writeback.io.data(1).valid := io.cpu.req_data.valid writeback.io.data(1).bits := io.cpu.req_data.bits - data.io.req_data.valid := io.cpu.req_data.valid && writeback.io.req(1).ready memCmdArb.io.in(0) <> mshr.io.mem.req_cmd memCmdArb.io.in(1) <> writeback.io.mem.req_cmd dataArb.io.in(0) <> mshr.io.data - dataArb.io.in(1).valid := s2_valid && s2_hit + dataArb.io.in(1).valid := s2_valid && s2_hit && writeback.io.req(0).ready && mshr.io.cpu.ready // stall_s2 dataArb.io.in(1).bits := s2 dataArb.io.in(1).bits.way := s2_hit_way dataArb.io.in(1).bits.isWriteback := Bool(false) - stall_s2 := s2_valid && !Mux(s2_hit, dataArb.io.in(1).ready, Mux(s2.rw, writeback.io.req(1).ready, mshr.io.cpu.ready)) + stall_s2 := s2_valid && !(dataArb.io.in(1).ready && writeback.io.req(0).ready && mshr.io.cpu.ready) io.cpu.resp <> data.io.resp io.cpu.req_cmd.ready := !stall_s1 && !replay_s1 - io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready && writeback.io.req(1).ready + io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready io.mem.req_cmd <> memCmdArb.io.out io.mem.req_data <> writeback.io.mem.req_data } From 17dc2075dd2c73b9446eb669a41d45aefd718ff1 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 6 Aug 2012 17:10:04 -0700 Subject: [PATCH 125/374] fix some LLC control bugs --- uncore/llc.scala | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index eece03ed..64d48ce8 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -383,15 +383,20 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val repl_way = LFSR16(s2_valid)(log2Up(ways)-1, 0) val repl_tag = s2_tags(repl_way).toUFix val setDirty = s2_valid && s2.rw && s2_hit && !s2_hit_dirty - stall_s1 := initialize || mshr.io.tag.valid || setDirty || s2_valid && !s2_hit || stall_s2 + stall_s1 := initialize || stall_s2 - tags.io.in.valid := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || setDirty || mshr.io.tag.valid - tags.io.in.bits.addr := Mux(initialize, initCount, Mux(setDirty, s2.addr, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)))(log2Up(sets)-1,0)) - tags.io.in.bits.rw := initialize || setDirty || mshr.io.tag.valid - tags.io.in.bits.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) - tags.io.in.bits.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way)))) + val tag_we = setDirty || mshr.io.tag.valid + val tag_waddr = Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(log2Up(sets)-1,0) + val tag_wdata = Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)) + val tag_wway = Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way) + tags.io.in.valid := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || tag_we + tags.io.in.bits.addr := Mux(initialize, initCount, Mux(tag_we, tag_waddr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)(log2Up(sets)-1,0))) + tags.io.in.bits.rw := initialize || tag_we + tags.io.in.bits.wdata := Mux(initialize, UFix(0), Fill(ways, tag_wdata)) + tags.io.in.bits.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(tag_wway))) + when (tag_we && Mux(stall_s2, s2.addr, s1.addr)(log2Up(sets)-1,0) === tag_waddr) { s2_tags(tag_wway) := tag_wdata } - mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw + mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw && dataArb.io.in(1).ready && writeback.io.req(0).ready // stall_s2 mshr.io.cpu.bits := s2 mshr.io.repl_way := repl_way mshr.io.repl_dirty := repl_tag(tagWidth+1, tagWidth).andR @@ -404,29 +409,29 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da data.io.mem_resp_set := mshr.io.mem_resp_set data.io.mem_resp_way := mshr.io.mem_resp_way data.io.req_data.bits := io.cpu.req_data.bits + data.io.req_data.valid := io.cpu.req_data.valid writeback.io.req(0) <> data.io.writeback writeback.io.data(0) <> data.io.writeback_data - writeback.io.req(1).valid := s2_valid && !s2_hit && s2.rw + writeback.io.req(1).valid := s2_valid && !s2_hit && s2.rw && dataArb.io.in(1).ready && mshr.io.cpu.ready // stall_s2 writeback.io.req(1).bits := s2.addr writeback.io.data(1).valid := io.cpu.req_data.valid writeback.io.data(1).bits := io.cpu.req_data.bits - data.io.req_data.valid := io.cpu.req_data.valid && writeback.io.req(1).ready memCmdArb.io.in(0) <> mshr.io.mem.req_cmd memCmdArb.io.in(1) <> writeback.io.mem.req_cmd dataArb.io.in(0) <> mshr.io.data - dataArb.io.in(1).valid := s2_valid && s2_hit + dataArb.io.in(1).valid := s2_valid && s2_hit && writeback.io.req(0).ready && mshr.io.cpu.ready // stall_s2 dataArb.io.in(1).bits := s2 dataArb.io.in(1).bits.way := s2_hit_way dataArb.io.in(1).bits.isWriteback := Bool(false) - stall_s2 := s2_valid && !Mux(s2_hit, dataArb.io.in(1).ready, Mux(s2.rw, writeback.io.req(1).ready, mshr.io.cpu.ready)) + stall_s2 := s2_valid && !(dataArb.io.in(1).ready && writeback.io.req(0).ready && mshr.io.cpu.ready) io.cpu.resp <> data.io.resp io.cpu.req_cmd.ready := !stall_s1 && !replay_s1 - io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready && writeback.io.req(1).ready + io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready io.mem.req_cmd <> memCmdArb.io.out io.mem.req_data <> writeback.io.mem.req_data } From aa7fd1f40b8fb86b48f4d753e3dc5f72cfa7aaaa Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 8 Aug 2012 22:11:32 -0700 Subject: [PATCH 126/374] rename queue to Queue fixes build with case-insensitive file system --- uncore/llc.scala | 2 +- uncore/uncore.scala | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 64d48ce8..a2a5d361 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -245,7 +245,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Compo val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] } - val q = (new queue(latency+2)) { new QEntry } + val q = (new Queue(latency+2)) { new QEntry } val qReady = q.io.count <= UFix(q.entries-latency-1) val valid = Reg(resetVal = Bool(false)) val req = Reg() { io.req.bits.clone } diff --git a/uncore/uncore.scala b/uncore/uncore.scala index a1726242..7ffef155 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -336,8 +336,8 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH } } - val p_rep_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY - val x_init_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY + val p_rep_data_dep_list = List.fill(ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY + val x_init_data_dep_list = List.fill(ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY // Free finished transactions for( j <- 0 until ntiles ) { From 6546dc84e2e53f5ec4e5b261217981e179f67ce6 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 8 Aug 2012 22:11:32 -0700 Subject: [PATCH 127/374] rename queue to Queue fixes build with case-insensitive file system --- uncore/llc.scala | 2 +- uncore/memserdes.scala | 2 +- uncore/slowio.scala | 4 ++-- uncore/uncore.scala | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 64d48ce8..a2a5d361 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -245,7 +245,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Compo val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] } - val q = (new queue(latency+2)) { new QEntry } + val q = (new Queue(latency+2)) { new QEntry } val qReady = q.io.count <= UFix(q.entries-latency-1) val valid = Reg(resetVal = Bool(false)) val req = Reg() { io.req.bits.clone } diff --git a/uncore/memserdes.scala b/uncore/memserdes.scala index e20f1ec3..cdd109c2 100644 --- a/uncore/memserdes.scala +++ b/uncore/memserdes.scala @@ -149,7 +149,7 @@ class MemDessert extends Component // test rig side io.wide.req_data.valid := state === s_data io.wide.req_data.bits.data := in_buf >> UFix(((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH - (dbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH)*MEM_BACKUP_WIDTH) - val dataq = (new queue(REFILL_CYCLES)) { new MemResp } + val dataq = (new Queue(REFILL_CYCLES)) { new MemResp } dataq.io.enq <> io.wide.resp dataq.io.deq.ready := recv_cnt === UFix((rbits-1)/MEM_BACKUP_WIDTH) diff --git a/uncore/slowio.scala b/uncore/slowio.scala index 6cf5a3d9..068e90c5 100644 --- a/uncore/slowio.scala +++ b/uncore/slowio.scala @@ -28,12 +28,12 @@ class slowIO[T <: Data](val divisor: Int, hold_cycles_in: Int = -1)(data: => T) val out_slow_val = Reg(resetVal = Bool(false)) val out_slow_bits = Reg() { data } - val fromhost_q = new queue(1)(data) + val fromhost_q = new Queue(1)(data) fromhost_q.io.enq.valid := in_en && (io.in_slow.valid && in_slow_rdy || reset) fromhost_q.io.enq.bits := io.in_slow.bits fromhost_q.io.deq <> io.in_fast - val tohost_q = new queue(1)(data) + val tohost_q = new Queue(1)(data) tohost_q.io.enq <> io.out_fast tohost_q.io.deq.ready := in_en && io.out_slow.ready && out_slow_val diff --git a/uncore/uncore.scala b/uncore/uncore.scala index a1726242..7ffef155 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -336,8 +336,8 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH } } - val p_rep_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY - val x_init_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY + val p_rep_data_dep_list = List.fill(ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY + val x_init_data_dep_list = List.fill(ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY // Free finished transactions for( j <- 0 until ntiles ) { From fa8075570ab7a5f1fc28ca3c8695dcad5997ef36 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Thu, 27 Sep 2012 12:59:45 -0700 Subject: [PATCH 128/374] move srcs into src dir, factoring out uncore consts into consts --- uncore/{ => src}/coherence.scala | 2 +- uncore/src/consts.scala | 28 ++++++++++++++++++++++++++++ uncore/{ => src}/llc.scala | 2 +- uncore/{ => src}/uncore.scala | 2 +- 4 files changed, 31 insertions(+), 3 deletions(-) rename uncore/{ => src}/coherence.scala (99%) create mode 100644 uncore/src/consts.scala rename uncore/{ => src}/llc.scala (99%) rename uncore/{ => src}/uncore.scala (99%) diff --git a/uncore/coherence.scala b/uncore/src/coherence.scala similarity index 99% rename from uncore/coherence.scala rename to uncore/src/coherence.scala index 04843fec..e4f85436 100644 --- a/uncore/coherence.scala +++ b/uncore/src/coherence.scala @@ -1,4 +1,4 @@ -package rocket +package uncore import Chisel._ import Constants._ diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala new file mode 100644 index 00000000..8ebf5c49 --- /dev/null +++ b/uncore/src/consts.scala @@ -0,0 +1,28 @@ +package uncore + +import Chisel._ + +object Constants +{ + + val X_INIT_TYPE_MAX_BITS = 2 + val X_REP_TYPE_MAX_BITS = 3 + val P_REQ_TYPE_MAX_BITS = 2 + val P_REP_TYPE_MAX_BITS = 3 + + val PADDR_BITS = 40; + val VADDR_BITS = 43; + val PGIDX_BITS = 13; + val PPN_BITS = PADDR_BITS-PGIDX_BITS; + val VPN_BITS = VADDR_BITS-PGIDX_BITS; + val ASID_BITS = 7; + val PERM_BITS = 6; + + + val COHERENCE_DATA_BITS = (1 << OFFSET_BITS)*8 + val TILE_ID_BITS = log2Up(NTILES)+1 + val TILE_XACT_ID_BITS = log2Up(NMSHR)+3 + val NGLOBAL_XACTS = 8 + val GLOBAL_XACT_ID_BITS = log2Up(NGLOBAL_XACTS) + +} diff --git a/uncore/llc.scala b/uncore/src/llc.scala similarity index 99% rename from uncore/llc.scala rename to uncore/src/llc.scala index a2a5d361..2bb2b62a 100644 --- a/uncore/llc.scala +++ b/uncore/src/llc.scala @@ -1,4 +1,4 @@ -package rocket +package uncore import Chisel._ import Node._ diff --git a/uncore/uncore.scala b/uncore/src/uncore.scala similarity index 99% rename from uncore/uncore.scala rename to uncore/src/uncore.scala index 7ffef155..56acece0 100644 --- a/uncore/uncore.scala +++ b/uncore/src/uncore.scala @@ -1,4 +1,4 @@ -package rocket +package uncore import Chisel._ import Constants._ From da6ec486f1e2cfbb0eb501c29262e05d3e52014d Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 27 Sep 2012 16:46:36 -0700 Subject: [PATCH 129/374] uncore and rocket changes for new xact types --- uncore/uncore.scala | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 7ffef155..84b74366 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -3,14 +3,17 @@ package rocket import Chisel._ import Constants._ +class PhysicalAddress extends Bundle { + val addr = UFix(width = PADDR_BITS - OFFSET_BITS) +} + class MemData extends Bundle { val data = Bits(width = MEM_DATA_BITS) } -class MemReqCmd() extends Bundle +class MemReqCmd() extends PhysicalAddress { val rw = Bool() - val addr = UFix(width = PADDR_BITS - OFFSET_BITS) val tag = Bits(width = MEM_TAG_BITS) } @@ -165,7 +168,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { io.probe_req.valid := Bool(false) io.probe_req.bits.p_type := co.getProbeRequestType(x_type_, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) - io.probe_req.bits.address := addr_ + io.probe_req.bits.addr := addr_ io.push_p_req := Bits(0, width = ntiles) io.pop_p_rep := Bits(0, width = ntiles) io.pop_p_rep_data := Bits(0, width = ntiles) @@ -178,7 +181,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { switch (state) { is(s_idle) { when( io.alloc_req.valid && io.can_alloc ) { - addr_ := io.alloc_req.bits.xact_init.address + addr_ := io.alloc_req.bits.xact_init.addr x_type_ := io.alloc_req.bits.xact_init.x_type init_tile_id_ := io.alloc_req.bits.tile_id tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id @@ -272,7 +275,7 @@ class CoherenceHubNull(co: ThreeStateIncoherence) extends CoherenceHub(1, co) io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write io.mem.req_cmd.bits.tag := x_init.bits.tile_xact_id - io.mem.req_cmd.bits.addr := x_init.bits.address + io.mem.req_cmd.bits.addr := x_init.bits.addr io.mem.req_data <> io.tiles(0).xact_init_data val x_rep = io.tiles(0).xact_rep @@ -432,7 +435,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH val conflicts = Vec(NGLOBAL_XACTS) { Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.address) + conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.addr) } x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits))) From 2413763f3dfb21acb1c05b1d37ebc34692d1a54e Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Mon, 1 Oct 2012 16:05:37 -0700 Subject: [PATCH 130/374] henry's uncore and rocket changes for new xact types --- uncore/src/coherence.scala | 194 +++++++++++++++++++++++++++---------- uncore/src/consts.scala | 60 +++++++++++- uncore/src/uncore.scala | 15 +-- 3 files changed, 205 insertions(+), 64 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index e4f85436..7fc03ece 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -3,10 +3,39 @@ package uncore import Chisel._ import Constants._ -class TransactionInit extends Bundle { +object TransactionInit +{ + def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix) = { + val init = new TransactionInit + init.x_type := x_type + init.addr := addr + init.tile_xact_id := tile_xact_id + init + } + def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix, write_mask: Bits) = { + val init = new TransactionInit + init.x_type := x_type + init.addr := addr + init.tile_xact_id := tile_xact_id + init.write_mask := write_mask + init + } + def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix, subword_addr: UFix, atomic_opcode: UFix) = { + val init = new TransactionInit + init.x_type := x_type + init.addr := addr + init.tile_xact_id := tile_xact_id + init.subword_addr := subword_addr + init.atomic_opcode := atomic_opcode + init + } +} +class TransactionInit extends PhysicalAddress { val x_type = Bits(width = X_INIT_TYPE_MAX_BITS) val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) - val address = UFix(width = PADDR_BITS - OFFSET_BITS) + val write_mask = Bits(width = X_INIT_WRITE_MASK_BITS) + val subword_addr = Bits(width = X_INIT_SUBWORD_ADDR_BITS) + val atomic_opcode = Bits(width = X_INIT_ATOMIC_OP_BITS) } class TransactionInitData extends MemData @@ -15,10 +44,9 @@ class TransactionAbort extends Bundle { val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) } -class ProbeRequest extends Bundle { +class ProbeRequest extends PhysicalAddress { val p_type = Bits(width = P_REQ_TYPE_MAX_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) - val address = Bits(width = PADDR_BITS - OFFSET_BITS) } class ProbeReply extends Bundle { @@ -76,6 +104,7 @@ abstract class CoherencePolicy { def messageHasData (init: TransactionInit): Bool def messageHasData (reply: TransactionReply): Bool def messageUpdatesDataArray (reply: TransactionReply): Bool + def messageIsUncached(init: TransactionInit): Bool def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool def getTransactionReplyType(x_type: UFix, count: UFix): Bits @@ -86,8 +115,11 @@ abstract class CoherencePolicy { } trait UncachedTransactions { - def getTransactionInitTypeOnUncachedRead(): UFix - def getTransactionInitTypeOnUncachedWrite(): UFix + def getUncachedReadTransactionInit(addr: UFix, id: UFix): TransactionInit + def getUncachedWriteTransactionInit(addr: UFix, id: UFix): TransactionInit + def getUncachedReadWordTransactionInit(addr: UFix, id: UFix): TransactionInit + def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits): TransactionInit + def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix): TransactionInit } abstract class CoherencePolicyWithUncached extends CoherencePolicy with UncachedTransactions @@ -115,6 +147,8 @@ class ThreeStateIncoherence extends IncoherentPolicy { val xactInitReadClean :: xactInitReadDirty :: xactInitWriteback :: Nil = Enum(3){ UFix() } val xactReplyData :: xactReplyAck :: Nil = Enum(2){ UFix() } val probeRepInvalidateAck :: Nil = Enum(1){ UFix() } + val uncachedTypeList = List() + val hasDataTypeList = List(xactInitWriteback) def isHit ( cmd: Bits, state: UFix): Bool = (state === tileClean || state === tileDirty) def isValid (state: UFix): Bool = state != tileInvalid @@ -149,9 +183,10 @@ class ThreeStateIncoherence extends IncoherentPolicy { def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteback //TODO def getTransactionInitTypeOnWriteback(): Bits = xactInitWriteback - def messageHasData (init: TransactionInit): Bool = (init.x_type === xactInitWriteback) + def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_) def messageHasData (reply: TransactionReply) = (reply.x_type === xactReplyData) def messageUpdatesDataArray (reply: TransactionReply) = (reply.x_type === xactReplyData) + def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_) } class MICoherence extends CoherencePolicyWithUncached { @@ -159,10 +194,12 @@ class MICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileValid :: Nil = Enum(2){ UFix() } val globalInvalid :: globalValid :: Nil = Enum(2){ UFix() } - val xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: Nil = Enum(3){ UFix() } - val xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: Nil = Enum(3){ UFix() } + val xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: Nil = Enum(6){ UFix() } + val xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: Nil = Enum(6){ UFix() } val probeReqInvalidate :: probeReqCopy :: Nil = Enum(2){ UFix() } val probeRepInvalidateData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepCopyAck :: Nil = Enum(4){ UFix() } + val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactReplyReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = state != tileInvalid def isValid (state: UFix): Bool = state != tileInvalid @@ -191,7 +228,10 @@ class MICoherence extends CoherencePolicyWithUncached { MuxLookup(incoming.x_type, tileInvalid, Array( xactReplyReadExclusive -> tileValid, xactReplyReadUncached -> tileInvalid, - xactReplyWriteUncached -> tileInvalid + xactReplyWriteUncached -> tileInvalid, + xactReplyReadWordUncached -> tileInvalid, + xactReplyWriteWordUncached -> tileInvalid, + xactReplyAtomicUncached -> tileInvalid )) } def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { @@ -201,8 +241,12 @@ class MICoherence extends CoherencePolicyWithUncached { )) } - def getTransactionInitTypeOnUncachedRead() = xactInitReadUncached - def getTransactionInitTypeOnUncachedWrite() = xactInitWriteUncached + def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id) + def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id) + def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) + def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) + def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = xactInitReadExclusive def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = xactInitReadExclusive def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached @@ -227,15 +271,14 @@ class MICoherence extends CoherencePolicyWithUncached { (reply.p_type === probeRepInvalidateData || reply.p_type === probeRepCopyData) } - def messageHasData (init: TransactionInit): Bool = { - (init.x_type === xactInitWriteUncached) - } + def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_) def messageHasData (reply: TransactionReply): Bool = { - (reply.x_type != xactReplyWriteUncached) + (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyWriteWordUncached) } def messageUpdatesDataArray (reply: TransactionReply): Bool = { (reply.x_type === xactReplyReadExclusive) } + def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) @@ -243,7 +286,10 @@ class MICoherence extends CoherencePolicyWithUncached { MuxLookup(x_type, xactReplyReadUncached, Array( xactInitReadExclusive -> xactReplyReadExclusive, xactInitReadUncached -> xactReplyReadUncached, - xactInitWriteUncached -> xactReplyWriteUncached + xactInitWriteUncached -> xactReplyWriteUncached, + xactInitReadWordUncached -> xactReplyReadWordUncached, + xactInitWriteWordUncached -> xactReplyWriteWordUncached, + xactInitAtomicUncached -> xactReplyAtomicUncached )) } @@ -251,7 +297,10 @@ class MICoherence extends CoherencePolicyWithUncached { MuxLookup(x_type, probeReqCopy, Array( xactInitReadExclusive -> probeReqInvalidate, xactInitReadUncached -> probeReqCopy, - xactInitWriteUncached -> probeReqInvalidate + xactInitWriteUncached -> probeReqInvalidate, + xactInitReadWordUncached -> probeReqCopy, + xactInitWriteWordUncached -> probeReqInvalidate, + xactInitAtomicUncached -> probeReqInvalidate )) } @@ -271,17 +320,19 @@ class MEICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(3){ UFix() } val globalInvalid :: globalExclusiveClean :: Nil = Enum(2){ UFix() } - val xactInitReadExclusiveClean :: xactInitReadExclusiveDirty :: xactInitReadUncached :: xactInitWriteUncached :: Nil = Enum(4){ UFix() } - val xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: Nil = Enum(4){ UFix() } + val xactInitReadExclusiveClean :: xactInitReadExclusiveDirty :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: Nil = Enum(7){ UFix() } + val xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: Nil = Enum(7){ UFix() } val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() } val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() } + val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactReplyReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = state != tileInvalid def isValid (state: UFix): Bool = state != tileInvalid def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { val (read, write) = cpuCmdToRW(cmd) - (read && (outstanding.x_type === xactInitReadUncached || outstanding.x_type === xactInitWriteUncached)) || + (read && messageIsUncached(outstanding)) || (write && (outstanding.x_type != xactInitReadExclusiveDirty)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { @@ -311,7 +362,10 @@ class MEICoherence extends CoherencePolicyWithUncached { xactReplyReadExclusive -> Mux(outstanding.x_type === xactInitReadExclusiveDirty, tileExclusiveDirty, tileExclusiveClean), xactReplyReadExclusiveAck -> tileExclusiveDirty, xactReplyReadUncached -> tileInvalid, - xactReplyWriteUncached -> tileInvalid + xactReplyWriteUncached -> tileInvalid, + xactReplyReadWordUncached -> tileInvalid, + xactReplyWriteWordUncached -> tileInvalid, + xactReplyAtomicUncached -> tileInvalid )) } def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { @@ -322,8 +376,12 @@ class MEICoherence extends CoherencePolicyWithUncached { )) } - def getTransactionInitTypeOnUncachedRead() = xactInitReadUncached - def getTransactionInitTypeOnUncachedWrite() = xactInitWriteUncached + def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id) + def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id) + def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) + def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) + def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) Mux(write, xactInitReadExclusiveDirty, xactInitReadExclusiveClean) @@ -357,15 +415,14 @@ class MEICoherence extends CoherencePolicyWithUncached { reply.p_type === probeRepDowngradeData || reply.p_type === probeRepCopyData) } - def messageHasData (init: TransactionInit): Bool = { - (init.x_type === xactInitWriteUncached) - } + def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_) def messageHasData (reply: TransactionReply): Bool = { - (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck) + (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck && reply.x_type != xactReplyWriteWordUncached) } def messageUpdatesDataArray (reply: TransactionReply): Bool = { (reply.x_type === xactReplyReadExclusive) } + def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) @@ -374,7 +431,10 @@ class MEICoherence extends CoherencePolicyWithUncached { xactInitReadExclusiveClean -> xactReplyReadExclusive, xactInitReadExclusiveDirty -> xactReplyReadExclusive, xactInitReadUncached -> xactReplyReadUncached, - xactInitWriteUncached -> xactReplyWriteUncached + xactInitWriteUncached -> xactReplyWriteUncached, + xactInitReadWordUncached -> xactReplyReadWordUncached, + xactInitWriteWordUncached -> xactReplyWriteWordUncached, + xactInitAtomicUncached -> xactReplyAtomicUncached )) } @@ -383,7 +443,10 @@ class MEICoherence extends CoherencePolicyWithUncached { xactInitReadExclusiveClean -> probeReqInvalidate, xactInitReadExclusiveDirty -> probeReqInvalidate, xactInitReadUncached -> probeReqCopy, - xactInitWriteUncached -> probeReqInvalidate + xactInitWriteUncached -> probeReqInvalidate, + xactInitReadWordUncached -> probeReqCopy, + xactInitWriteWordUncached -> probeReqInvalidate, + xactInitAtomicUncached -> probeReqInvalidate )) } @@ -403,10 +466,12 @@ class MSICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileShared :: tileExclusiveDirty :: Nil = Enum(3){ UFix() } val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(3){ UFix() } - val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: Nil = Enum(4){ UFix() } - val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: Nil = Enum(5){ UFix() } + val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: Nil = Enum(7){ UFix() } + val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: Nil = Enum(8){ UFix() } val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() } val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() } + val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactReplyReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) @@ -419,7 +484,7 @@ class MSICoherence extends CoherencePolicyWithUncached { def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { val (read, write) = cpuCmdToRW(cmd) - (read && (outstanding.x_type === xactInitReadUncached || outstanding.x_type === xactInitWriteUncached)) || + (read && messageIsUncached(outstanding)) || (write && (outstanding.x_type != xactInitReadExclusive)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { @@ -450,7 +515,10 @@ class MSICoherence extends CoherencePolicyWithUncached { xactReplyReadExclusive -> tileExclusiveDirty, xactReplyReadExclusiveAck -> tileExclusiveDirty, xactReplyReadUncached -> tileInvalid, - xactReplyWriteUncached -> tileInvalid + xactReplyWriteUncached -> tileInvalid, + xactReplyReadWordUncached -> tileInvalid, + xactReplyWriteWordUncached -> tileInvalid, + xactReplyAtomicUncached -> tileInvalid )) } def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { @@ -461,8 +529,12 @@ class MSICoherence extends CoherencePolicyWithUncached { )) } - def getTransactionInitTypeOnUncachedRead() = xactInitReadUncached - def getTransactionInitTypeOnUncachedWrite() = xactInitWriteUncached + def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id) + def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id) + def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) + def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) + def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) Mux(write || cmd === M_PFW, xactInitReadExclusive, xactInitReadShared) @@ -496,15 +568,14 @@ class MSICoherence extends CoherencePolicyWithUncached { reply.p_type === probeRepDowngradeData || reply.p_type === probeRepCopyData) } - def messageHasData (init: TransactionInit): Bool = { - (init.x_type === xactInitWriteUncached) - } + def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_) def messageHasData (reply: TransactionReply): Bool = { - (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck) + (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck && reply.x_type != xactReplyWriteWordUncached) } def messageUpdatesDataArray (reply: TransactionReply): Bool = { (reply.x_type === xactReplyReadShared || reply.x_type === xactReplyReadExclusive) } + def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) @@ -513,7 +584,10 @@ class MSICoherence extends CoherencePolicyWithUncached { xactInitReadShared -> Mux(count > UFix(0), xactReplyReadShared, xactReplyReadExclusive), xactInitReadExclusive -> xactReplyReadExclusive, xactInitReadUncached -> xactReplyReadUncached, - xactInitWriteUncached -> xactReplyWriteUncached + xactInitWriteUncached -> xactReplyWriteUncached, + xactInitReadWordUncached -> xactReplyReadWordUncached, + xactInitWriteWordUncached -> xactReplyWriteWordUncached, + xactInitAtomicUncached -> xactReplyAtomicUncached )) } @@ -542,10 +616,12 @@ class MESICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(4){ UFix() } val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(3){ UFix() } - val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: Nil = Enum(4){ UFix() } - val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: Nil = Enum(5){ UFix() } + val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: Nil = Enum(7){ UFix() } + val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: Nil = Enum(8){ UFix() } val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() } val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() } + val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactReplyReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) @@ -558,7 +634,7 @@ class MESICoherence extends CoherencePolicyWithUncached { def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { val (read, write) = cpuCmdToRW(cmd) - (read && (outstanding.x_type === xactInitReadUncached || outstanding.x_type === xactInitWriteUncached)) || + (read && messageIsUncached(outstanding)) || (write && (outstanding.x_type != xactInitReadExclusive)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { @@ -589,7 +665,10 @@ class MESICoherence extends CoherencePolicyWithUncached { xactReplyReadExclusive -> Mux(outstanding.x_type === xactInitReadExclusive, tileExclusiveDirty, tileExclusiveClean), xactReplyReadExclusiveAck -> tileExclusiveDirty, xactReplyReadUncached -> tileInvalid, - xactReplyWriteUncached -> tileInvalid + xactReplyWriteUncached -> tileInvalid, + xactReplyReadWordUncached -> tileInvalid, + xactReplyWriteWordUncached -> tileInvalid, + xactReplyAtomicUncached -> tileInvalid )) } def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { @@ -600,8 +679,12 @@ class MESICoherence extends CoherencePolicyWithUncached { )) } - def getTransactionInitTypeOnUncachedRead() = xactInitReadUncached - def getTransactionInitTypeOnUncachedWrite() = xactInitWriteUncached + def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id) + def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id) + def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) + def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) + def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) Mux(write || cmd === M_PFW, xactInitReadExclusive, xactInitReadShared) @@ -635,15 +718,14 @@ class MESICoherence extends CoherencePolicyWithUncached { reply.p_type === probeRepDowngradeData || reply.p_type === probeRepCopyData) } - def messageHasData (init: TransactionInit): Bool = { - (init.x_type === xactInitWriteUncached) - } + def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_) def messageHasData (reply: TransactionReply): Bool = { - (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck) + (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck && reply.x_type != xactReplyWriteWordUncached) } def messageUpdatesDataArray (reply: TransactionReply): Bool = { (reply.x_type === xactReplyReadShared || reply.x_type === xactReplyReadExclusive) } + def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) @@ -652,7 +734,10 @@ class MESICoherence extends CoherencePolicyWithUncached { xactInitReadShared -> Mux(count > UFix(0), xactReplyReadShared, xactReplyReadExclusive), xactInitReadExclusive -> xactReplyReadExclusive, xactInitReadUncached -> xactReplyReadUncached, - xactInitWriteUncached -> xactReplyWriteUncached + xactInitWriteUncached -> xactReplyWriteUncached, + xactInitReadWordUncached -> xactReplyReadWordUncached, + xactInitWriteWordUncached -> xactReplyWriteWordUncached, + xactInitAtomicUncached -> xactReplyAtomicUncached )) } @@ -661,7 +746,10 @@ class MESICoherence extends CoherencePolicyWithUncached { xactInitReadShared -> probeReqDowngrade, xactInitReadExclusive -> probeReqInvalidate, xactInitReadUncached -> probeReqCopy, - xactInitWriteUncached -> probeReqInvalidate + xactInitWriteUncached -> probeReqInvalidate, + xactInitReadWordUncached -> probeReqCopy, + xactInitWriteWordUncached -> probeReqInvalidate, + xactInitAtomicUncached -> probeReqInvalidate )) } diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index 8ebf5c49..95b07cc8 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -1,15 +1,33 @@ package uncore import Chisel._ +import scala.math._ object Constants { + val NTILES = 1 + val HAVE_RVC = false + val HAVE_FPU = true + val HAVE_VEC = true - val X_INIT_TYPE_MAX_BITS = 2 - val X_REP_TYPE_MAX_BITS = 3 - val P_REQ_TYPE_MAX_BITS = 2 - val P_REP_TYPE_MAX_BITS = 3 - + val M_X = Bits("b????", 4); + val M_XRD = Bits("b0000", 4); // int load + val M_XWR = Bits("b0001", 4); // int store + val M_PFR = Bits("b0010", 4); // prefetch with intent to read + val M_PFW = Bits("b0011", 4); // prefetch with intent to write + val M_FLA = Bits("b0100", 4); // write back and invlaidate all lines + val M_FENCE = Bits("b0101", 4); // memory fence + val M_INV = Bits("b0110", 4); // write back and invalidate line + val M_CLN = Bits("b0111", 4); // write back line + val M_XA_ADD = Bits("b1000", 4); + val M_XA_SWAP = Bits("b1001", 4); + val M_XA_AND = Bits("b1010", 4); + val M_XA_OR = Bits("b1011", 4); + val M_XA_MIN = Bits("b1100", 4); + val M_XA_MAX = Bits("b1101", 4); + val M_XA_MINU = Bits("b1110", 4); + val M_XA_MAXU = Bits("b1111", 4); + val PADDR_BITS = 40; val VADDR_BITS = 43; val PGIDX_BITS = 13; @@ -18,6 +36,25 @@ object Constants val ASID_BITS = 7; val PERM_BITS = 6; + // rocketNBDCache parameters + val DCACHE_PORTS = 3 + val CPU_DATA_BITS = 64; + val CPU_TAG_BITS = 9; + val DCACHE_TAG_BITS = log2Up(DCACHE_PORTS) + CPU_TAG_BITS + val OFFSET_BITS = 6; // log2(cache line size in bytes) + val NMSHR = if (HAVE_VEC) 4 else 2 // number of primary misses + val NRPQ = 16; // number of secondary misses + val NSDQ = 17; // number of secondary stores/AMOs + val LG_REFILL_WIDTH = 4; // log2(cache bus width in bytes) + val IDX_BITS = 7; + val TAG_BITS = PADDR_BITS - OFFSET_BITS - IDX_BITS; + val NWAYS = 4 + require(IDX_BITS+OFFSET_BITS <= PGIDX_BITS); + + // coherence parameters + val ENABLE_SHARING = true + val ENABLE_CLEAN_EXCLUSIVE = true + val COHERENCE_DATA_BITS = (1 << OFFSET_BITS)*8 val TILE_ID_BITS = log2Up(NTILES)+1 @@ -25,4 +62,17 @@ object Constants val NGLOBAL_XACTS = 8 val GLOBAL_XACT_ID_BITS = log2Up(NGLOBAL_XACTS) + val X_INIT_TYPE_MAX_BITS = 2 + val X_INIT_WRITE_MASK_BITS = OFFSET_BITS + val X_INIT_SUBWORD_ADDR_BITS = log2Up(OFFSET_BITS) + val X_INIT_ATOMIC_OP_BITS = 4 + val X_REP_TYPE_MAX_BITS = 3 + val P_REQ_TYPE_MAX_BITS = 2 + val P_REP_TYPE_MAX_BITS = 3 + + // external memory interface + val MEM_TAG_BITS = max(TILE_XACT_ID_BITS, GLOBAL_XACT_ID_BITS) + val MEM_DATA_BITS = 128 + val REFILL_CYCLES = (1 << OFFSET_BITS)*8/MEM_DATA_BITS + } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 56acece0..1220a56f 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -3,14 +3,17 @@ package uncore import Chisel._ import Constants._ +class PhysicalAddress extends Bundle { + val addr = UFix(width = PADDR_BITS - OFFSET_BITS) +} + class MemData extends Bundle { val data = Bits(width = MEM_DATA_BITS) } -class MemReqCmd() extends Bundle +class MemReqCmd() extends PhysicalAddress { val rw = Bool() - val addr = UFix(width = PADDR_BITS - OFFSET_BITS) val tag = Bits(width = MEM_TAG_BITS) } @@ -165,7 +168,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { io.probe_req.valid := Bool(false) io.probe_req.bits.p_type := co.getProbeRequestType(x_type_, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) - io.probe_req.bits.address := addr_ + io.probe_req.bits.addr := addr_ io.push_p_req := Bits(0, width = ntiles) io.pop_p_rep := Bits(0, width = ntiles) io.pop_p_rep_data := Bits(0, width = ntiles) @@ -178,7 +181,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { switch (state) { is(s_idle) { when( io.alloc_req.valid && io.can_alloc ) { - addr_ := io.alloc_req.bits.xact_init.address + addr_ := io.alloc_req.bits.xact_init.addr x_type_ := io.alloc_req.bits.xact_init.x_type init_tile_id_ := io.alloc_req.bits.tile_id tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id @@ -272,7 +275,7 @@ class CoherenceHubNull(co: ThreeStateIncoherence) extends CoherenceHub(1, co) io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write io.mem.req_cmd.bits.tag := x_init.bits.tile_xact_id - io.mem.req_cmd.bits.addr := x_init.bits.address + io.mem.req_cmd.bits.addr := x_init.bits.addr io.mem.req_data <> io.tiles(0).xact_init_data val x_rep = io.tiles(0).xact_rep @@ -432,7 +435,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH val conflicts = Vec(NGLOBAL_XACTS) { Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.address) + conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.addr) } x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits))) From cf8f20584e21560b99970b8edffefa79569dd125 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Mon, 1 Oct 2012 16:08:41 -0700 Subject: [PATCH 131/374] factoring out uncore into separate uncore repo --- uncore/llc.scala | 437 ---------------------------------- uncore/memserdes.scala | 1 + uncore/uncore.scala | 519 ----------------------------------------- 3 files changed, 1 insertion(+), 956 deletions(-) delete mode 100644 uncore/llc.scala delete mode 100644 uncore/uncore.scala diff --git a/uncore/llc.scala b/uncore/llc.scala deleted file mode 100644 index a2a5d361..00000000 --- a/uncore/llc.scala +++ /dev/null @@ -1,437 +0,0 @@ -package rocket - -import Chisel._ -import Node._ -import Constants._ - -class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component -{ - class Inputs extends Bundle { - val addr = UFix(INPUT, log2Up(n)) - val rw = Bool(INPUT) - val wdata = gen.asInput - val wmask = gen.asInput - override def clone = new Inputs().asInstanceOf[this.type] - } - val io = new Bundle { - val in = new PipeIO()(new Inputs).flip - val rdata = gen.asOutput - } - val data = gen - val colMux = if (2*data.width <= leaf.data.width && n > leaf.n) 1 << math.floor(math.log(leaf.data.width/data.width)/math.log(2)).toInt else 1 - val nWide = if (data.width > leaf.data.width) 1+(data.width-1)/leaf.data.width else 1 - val nDeep = if (n > colMux*leaf.n) 1+(n-1)/(colMux*leaf.n) else 1 - if (nDeep > 1 || colMux > 1) - require(isPow2(n) && isPow2(leaf.n)) - - val rdataDeep = Vec(nDeep) { Bits() } - val rdataSel = Vec(nDeep) { Bool() } - for (i <- 0 until nDeep) { - val in = Pipe(io.in.valid && (if (nDeep == 1) Bool(true) else UFix(i) === io.in.bits.addr(log2Up(n)-1, log2Up(n/nDeep))), io.in.bits, preLatency) - val idx = in.bits.addr(log2Up(n/nDeep/colMux)-1, 0) - val wdata = in.bits.wdata.toBits - val wmask = in.bits.wmask.toBits - val ren = in.valid && !in.bits.rw - val reg_ren = Reg(ren) - val rdata = Vec(nWide) { Bits() } - - val r = Pipe(ren, in.bits.addr, postLatency) - - for (j <- 0 until nWide) { - val mem = leaf.clone - var dout: Bits = null - val dout1 = if (postLatency > 0) Reg() { Bits() } else null - - var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) - if (colMux > 1) - wmask0 = wmask0 & FillInterleaved(gen.width, UFixToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) - val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) - when (in.valid) { - when (in.bits.rw) { mem.write(idx, wdata0, wmask0) } - .otherwise { if (postLatency > 0) dout1 := mem(idx) } - } - - if (postLatency == 0) { - dout = mem(idx) - } else if (postLatency == 1) { - dout = dout1 - } else - dout = Pipe(reg_ren, dout1, postLatency-1).bits - - rdata(j) := dout - } - val rdataWide = rdata.reduceLeft((x, y) => Cat(y, x)) - - var colMuxOut = rdataWide - if (colMux > 1) { - val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.width*(k+1)-1, gen.width*k))) { Bits() } - colMuxOut = colMuxIn(r.bits(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) - } - - rdataDeep(i) := colMuxOut - rdataSel(i) := r.valid - } - - io.rdata := Mux1H(rdataSel, rdataDeep) -} - -class LLCDataReq(ways: Int) extends MemReqCmd -{ - val way = UFix(width = log2Up(ways)) - val isWriteback = Bool() - - override def clone = new LLCDataReq(ways).asInstanceOf[this.type] -} - -class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component -{ - val io = new Bundle { - val cpu = (new FIFOIO) { new MemReqCmd }.flip - val repl_way = UFix(INPUT, log2Up(ways)) - val repl_dirty = Bool(INPUT) - val repl_tag = UFix(INPUT, PADDR_BITS - OFFSET_BITS - log2Up(sets)) - val data = (new FIFOIO) { new LLCDataReq(ways) } - val tag = (new FIFOIO) { new Bundle { - val addr = UFix(width = PADDR_BITS - OFFSET_BITS) - val way = UFix(width = log2Up(ways)) - } } - val mem = new ioMemPipe - val mem_resp_set = UFix(OUTPUT, log2Up(sets)) - val mem_resp_way = UFix(OUTPUT, log2Up(ways)) - } - - class MSHR extends Bundle { - val addr = UFix(width = PADDR_BITS - OFFSET_BITS) - val way = UFix(width = log2Up(ways)) - val tag = io.cpu.bits.tag.clone - val refilled = Bool() - val refillCount = UFix(width = log2Up(REFILL_CYCLES)) - val requested = Bool() - val old_dirty = Bool() - val old_tag = UFix(width = PADDR_BITS - OFFSET_BITS - log2Up(sets)) - - override def clone = new MSHR().asInstanceOf[this.type] - } - - val valid = Vec(outstanding) { Reg(resetVal = Bool(false)) } - val validBits = valid.toBits - val freeId = PriorityEncoder(~validBits) - val mshr = Vec(outstanding) { Reg() { new MSHR } } - when (io.cpu.valid && io.cpu.ready) { - valid(freeId) := Bool(true) - mshr(freeId).addr := io.cpu.bits.addr - mshr(freeId).tag := io.cpu.bits.tag - mshr(freeId).way := io.repl_way - mshr(freeId).old_dirty := io.repl_dirty - mshr(freeId).old_tag := io.repl_tag - mshr(freeId).requested := Bool(false) - mshr(freeId).refillCount := UFix(0) - mshr(freeId).refilled := Bool(false) - } - - val requests = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && !mshr(i).old_dirty && !mshr(i).requested):_*) - val request = requests.orR - val requestId = PriorityEncoder(requests) - when (io.mem.req_cmd.valid && io.mem.req_cmd.ready) { mshr(requestId).requested := Bool(true) } - - val refillId = io.mem.resp.bits.tag(log2Up(outstanding)-1, 0) - val refillCount = mshr(refillId).refillCount - when (io.mem.resp.valid) { - mshr(refillId).refillCount := refillCount + UFix(1) - when (refillCount === UFix(REFILL_CYCLES-1)) { mshr(refillId).refilled := Bool(true) } - } - - val replays = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).refilled):_*) - val replay = replays.orR - val replayId = PriorityEncoder(replays) - when (replay && io.data.ready && io.tag.ready) { valid(replayId) := Bool(false) } - - val writebacks = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).old_dirty):_*) - val writeback = writebacks.orR - val writebackId = PriorityEncoder(writebacks) - when (writeback && io.data.ready && !replay) { mshr(writebackId).old_dirty := Bool(false) } - - val conflicts = Cat(Bits(0), (0 until outstanding).map(i => valid(i) && io.cpu.bits.addr(log2Up(sets)-1, 0) === mshr(i).addr(log2Up(sets)-1, 0)):_*) - io.cpu.ready := !conflicts.orR && !validBits.andR - - io.data.valid := writeback - io.data.bits.rw := Bool(false) - io.data.bits.tag := mshr(replayId).tag - io.data.bits.isWriteback := Bool(true) - io.data.bits.addr := Cat(mshr(writebackId).old_tag, mshr(writebackId).addr(log2Up(sets)-1, 0)).toUFix - io.data.bits.way := mshr(writebackId).way - when (replay) { - io.data.valid := io.tag.ready - io.data.bits.isWriteback := Bool(false) - io.data.bits.addr := mshr(replayId).addr - io.data.bits.way := mshr(replayId).way - } - io.tag.valid := replay && io.data.ready - io.tag.bits.addr := io.data.bits.addr - io.tag.bits.way := io.data.bits.way - - io.mem.req_cmd.valid := request - io.mem.req_cmd.bits.rw := Bool(false) - io.mem.req_cmd.bits.addr := mshr(requestId).addr - io.mem.req_cmd.bits.tag := requestId - io.mem_resp_set := mshr(refillId).addr - io.mem_resp_way := mshr(refillId).way -} - -class LLCWriteback(requestors: Int) extends Component -{ - val io = new Bundle { - val req = Vec(requestors) { (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) }.flip } - val data = Vec(requestors) { (new FIFOIO) { new MemData }.flip } - val mem = new ioMemPipe - } - - val valid = Reg(resetVal = Bool(false)) - val who = Reg() { UFix() } - val addr = Reg() { UFix() } - val cmd_sent = Reg() { Bool() } - val data_sent = Reg() { Bool() } - val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - - var anyReq = Bool(false) - for (i <- 0 until requestors) { - io.req(i).ready := !valid && !anyReq - io.data(i).ready := valid && who === UFix(i) && io.mem.req_data.ready - anyReq = anyReq || io.req(i).valid - } - - val nextWho = PriorityEncoder(io.req.map(_.valid)) - when (!valid && io.req.map(_.valid).reduceLeft(_||_)) { - valid := Bool(true) - cmd_sent := Bool(false) - data_sent := Bool(false) - who := nextWho - addr := io.req(nextWho).bits - } - - when (io.mem.req_data.valid && io.mem.req_data.ready) { - count := count + UFix(1) - when (count === UFix(REFILL_CYCLES-1)) { - data_sent := Bool(true) - when (cmd_sent) { valid := Bool(false) } - } - } - when (io.mem.req_cmd.valid && io.mem.req_cmd.ready) { cmd_sent := Bool(true) } - when (valid && cmd_sent && data_sent) { valid := Bool(false) } - - io.mem.req_cmd.valid := valid && !cmd_sent - io.mem.req_cmd.bits.addr := addr - io.mem.req_cmd.bits.rw := Bool(true) - - io.mem.req_data.valid := valid && !data_sent && io.data(who).valid - io.mem.req_data.bits := io.data(who).bits -} - -class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Component -{ - val io = new Bundle { - val req = (new FIFOIO) { new LLCDataReq(ways) }.flip - val req_data = (new FIFOIO) { new MemData }.flip - val writeback = (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) } - val writeback_data = (new FIFOIO) { new MemData } - val resp = (new FIFOIO) { new MemResp } - val mem_resp = (new PipeIO) { new MemResp }.flip - val mem_resp_set = UFix(INPUT, log2Up(sets)) - val mem_resp_way = UFix(INPUT, log2Up(ways)) - } - - val data = new BigMem(sets*ways*REFILL_CYCLES, 1, latency-1, leaf)(Bits(width = MEM_DATA_BITS)) - class QEntry extends MemResp { - val isWriteback = Bool() - override def clone = new QEntry().asInstanceOf[this.type] - } - val q = (new Queue(latency+2)) { new QEntry } - val qReady = q.io.count <= UFix(q.entries-latency-1) - val valid = Reg(resetVal = Bool(false)) - val req = Reg() { io.req.bits.clone } - val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - val refillCount = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - - when (data.io.in.valid && !io.mem_resp.valid) { - count := count + UFix(1) - when (valid && count === UFix(REFILL_CYCLES-1)) { valid := Bool(false) } - } - when (io.req.valid && io.req.ready) { valid := Bool(true); req := io.req.bits } - when (io.mem_resp.valid) { refillCount := refillCount + UFix(1) } - - data.io.in.valid := io.req.valid && io.req.ready && Mux(io.req.bits.rw, io.req_data.valid, qReady) - data.io.in.bits.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUFix - data.io.in.bits.rw := io.req.bits.rw - data.io.in.bits.wdata := io.req_data.bits.data - data.io.in.bits.wmask := Fix(-1, io.req_data.bits.data.width) - when (valid) { - data.io.in.valid := Mux(req.rw, io.req_data.valid, qReady) - data.io.in.bits.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUFix - data.io.in.bits.rw := req.rw - } - when (io.mem_resp.valid) { - data.io.in.valid := Bool(true) - data.io.in.bits.addr := Cat(io.mem_resp_way, io.mem_resp_set, refillCount).toUFix - data.io.in.bits.rw := Bool(true) - data.io.in.bits.wdata := io.mem_resp.bits.data - } - - val tagPipe = Pipe(data.io.in.valid && !data.io.in.bits.rw, Mux(valid, req.tag, io.req.bits.tag), latency) - q.io.enq.valid := tagPipe.valid - q.io.enq.bits.tag := tagPipe.bits - q.io.enq.bits.isWriteback := Pipe(Mux(valid, req.isWriteback, io.req.bits.isWriteback), Bool(false), latency).valid - q.io.enq.bits.data := data.io.rdata - - io.req.ready := !valid && Mux(io.req.bits.isWriteback, io.writeback.ready, Bool(true)) - io.req_data.ready := !io.mem_resp.valid && Mux(valid, req.rw, io.req.valid && io.req.bits.rw) - - io.writeback.valid := io.req.valid && io.req.ready && io.req.bits.isWriteback - io.writeback.bits := io.req.bits.addr - - q.io.deq.ready := Mux(q.io.deq.bits.isWriteback, io.writeback_data.ready, io.resp.ready) - io.resp.valid := q.io.deq.valid && !q.io.deq.bits.isWriteback - io.resp.bits := q.io.deq.bits - io.writeback_data.valid := q.io.deq.valid && q.io.deq.bits.isWriteback - io.writeback_data.bits := q.io.deq.bits -} - -class MemReqArb(n: Int) extends Component // UNTESTED -{ - val io = new Bundle { - val cpu = Vec(n) { new ioMem().flip } - val mem = new ioMem - } - - val lock = Reg(resetVal = Bool(false)) - val locker = Reg() { UFix() } - - val arb = new RRArbiter(n)(new MemReqCmd) - val respWho = io.mem.resp.bits.tag(log2Up(n)-1,0) - val respTag = io.mem.resp.bits.tag >> UFix(log2Up(n)) - for (i <- 0 until n) { - val me = UFix(i, log2Up(n)) - arb.io.in(i).valid := io.cpu(i).req_cmd.valid - arb.io.in(i).bits := io.cpu(i).req_cmd.bits - arb.io.in(i).bits.tag := Cat(io.cpu(i).req_cmd.bits.tag, me) - io.cpu(i).req_cmd.ready := arb.io.in(i).ready - io.cpu(i).req_data.ready := Bool(false) - - val getLock = io.cpu(i).req_cmd.fire() && io.cpu(i).req_cmd.bits.rw && !lock - val haveLock = lock && locker === me - when (getLock) { - lock := Bool(true) - locker := UFix(i) - } - when (getLock || haveLock) { - io.cpu(i).req_data.ready := io.mem.req_data.ready - io.mem.req_data.valid := Bool(true) - io.mem.req_data.bits := io.cpu(i).req_data.bits - } - - io.cpu(i).resp.valid := io.mem.resp.valid && respWho === me - io.cpu(i).resp.bits := io.mem.resp.bits - io.cpu(i).resp.bits.tag := respTag - } - io.mem.resp.ready := io.cpu(respWho).resp.ready - - val unlock = Counter(io.mem.req_data.fire(), REFILL_CYCLES)._2 - when (unlock) { lock := Bool(false) } -} - -class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits]) extends Component -{ - val io = new Bundle { - val cpu = new ioMem().flip - val mem = new ioMemPipe - } - - val tagWidth = PADDR_BITS - OFFSET_BITS - log2Up(sets) - val metaWidth = tagWidth + 2 // valid + dirty - - val memCmdArb = (new Arbiter(2)) { new MemReqCmd } - val dataArb = (new Arbiter(2)) { new LLCDataReq(ways) } - val mshr = new LLCMSHRFile(sets, ways, outstanding) - val tags = new BigMem(sets, 0, 1, tagLeaf)(Bits(width = metaWidth*ways)) - val data = new LLCData(4, sets, ways, dataLeaf) - val writeback = new LLCWriteback(2) - - val initCount = Reg(resetVal = UFix(0, log2Up(sets+1))) - val initialize = !initCount(log2Up(sets)) - when (initialize) { initCount := initCount + UFix(1) } - - val stall_s1 = Bool() - val replay_s1 = Reg(resetVal = Bool(false)) - val s1_valid = Reg(io.cpu.req_cmd.valid && !stall_s1 || replay_s1, resetVal = Bool(false)) - replay_s1 := s1_valid && stall_s1 - val s1 = Reg() { new MemReqCmd } - when (io.cpu.req_cmd.valid && io.cpu.req_cmd.ready) { s1 := io.cpu.req_cmd.bits } - - val stall_s2 = Bool() - val s2_valid = Reg(resetVal = Bool(false)) - s2_valid := s1_valid && !replay_s1 && !stall_s1 || stall_s2 - val s2 = Reg() { new MemReqCmd } - val s2_tags = Vec(ways) { Reg() { Bits(width = metaWidth) } } - when (s1_valid && !stall_s1 && !replay_s1) { - s2 := s1 - for (i <- 0 until ways) - s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) - } - val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.width-1, s2.addr.width-tagWidth) === t(tagWidth-1, 0)) - val s2_hit_way = OHToUFix(s2_hits) - val s2_hit = s2_hits.reduceLeft(_||_) - val s2_hit_dirty = s2_tags(s2_hit_way)(tagWidth+1) - val repl_way = LFSR16(s2_valid)(log2Up(ways)-1, 0) - val repl_tag = s2_tags(repl_way).toUFix - val setDirty = s2_valid && s2.rw && s2_hit && !s2_hit_dirty - stall_s1 := initialize || stall_s2 - - val tag_we = setDirty || mshr.io.tag.valid - val tag_waddr = Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(log2Up(sets)-1,0) - val tag_wdata = Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)) - val tag_wway = Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way) - tags.io.in.valid := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || tag_we - tags.io.in.bits.addr := Mux(initialize, initCount, Mux(tag_we, tag_waddr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)(log2Up(sets)-1,0))) - tags.io.in.bits.rw := initialize || tag_we - tags.io.in.bits.wdata := Mux(initialize, UFix(0), Fill(ways, tag_wdata)) - tags.io.in.bits.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(tag_wway))) - when (tag_we && Mux(stall_s2, s2.addr, s1.addr)(log2Up(sets)-1,0) === tag_waddr) { s2_tags(tag_wway) := tag_wdata } - - mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw && dataArb.io.in(1).ready && writeback.io.req(0).ready // stall_s2 - mshr.io.cpu.bits := s2 - mshr.io.repl_way := repl_way - mshr.io.repl_dirty := repl_tag(tagWidth+1, tagWidth).andR - mshr.io.repl_tag := repl_tag - mshr.io.mem.resp := io.mem.resp - mshr.io.tag.ready := !setDirty - - data.io.req <> dataArb.io.out - data.io.mem_resp := io.mem.resp - data.io.mem_resp_set := mshr.io.mem_resp_set - data.io.mem_resp_way := mshr.io.mem_resp_way - data.io.req_data.bits := io.cpu.req_data.bits - data.io.req_data.valid := io.cpu.req_data.valid - - writeback.io.req(0) <> data.io.writeback - writeback.io.data(0) <> data.io.writeback_data - writeback.io.req(1).valid := s2_valid && !s2_hit && s2.rw && dataArb.io.in(1).ready && mshr.io.cpu.ready // stall_s2 - writeback.io.req(1).bits := s2.addr - writeback.io.data(1).valid := io.cpu.req_data.valid - writeback.io.data(1).bits := io.cpu.req_data.bits - - memCmdArb.io.in(0) <> mshr.io.mem.req_cmd - memCmdArb.io.in(1) <> writeback.io.mem.req_cmd - - dataArb.io.in(0) <> mshr.io.data - dataArb.io.in(1).valid := s2_valid && s2_hit && writeback.io.req(0).ready && mshr.io.cpu.ready // stall_s2 - dataArb.io.in(1).bits := s2 - dataArb.io.in(1).bits.way := s2_hit_way - dataArb.io.in(1).bits.isWriteback := Bool(false) - - stall_s2 := s2_valid && !(dataArb.io.in(1).ready && writeback.io.req(0).ready && mshr.io.cpu.ready) - - io.cpu.resp <> data.io.resp - io.cpu.req_cmd.ready := !stall_s1 && !replay_s1 - io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready - io.mem.req_cmd <> memCmdArb.io.out - io.mem.req_data <> writeback.io.mem.req_data -} diff --git a/uncore/memserdes.scala b/uncore/memserdes.scala index cdd109c2..712dec16 100644 --- a/uncore/memserdes.scala +++ b/uncore/memserdes.scala @@ -4,6 +4,7 @@ import Chisel._ import Node._ import Constants._ import scala.math._ +import uncore._ class ioMemSerialized extends Bundle { diff --git a/uncore/uncore.scala b/uncore/uncore.scala deleted file mode 100644 index 84b74366..00000000 --- a/uncore/uncore.scala +++ /dev/null @@ -1,519 +0,0 @@ -package rocket - -import Chisel._ -import Constants._ - -class PhysicalAddress extends Bundle { - val addr = UFix(width = PADDR_BITS - OFFSET_BITS) -} - -class MemData extends Bundle { - val data = Bits(width = MEM_DATA_BITS) -} - -class MemReqCmd() extends PhysicalAddress -{ - val rw = Bool() - val tag = Bits(width = MEM_TAG_BITS) -} - -class MemResp () extends MemData -{ - val tag = Bits(width = MEM_TAG_BITS) -} - -class ioMem() extends Bundle -{ - val req_cmd = (new FIFOIO) { new MemReqCmd() } - val req_data = (new FIFOIO) { new MemData() } - val resp = (new FIFOIO) { new MemResp() }.flip -} - -class ioMemPipe() extends Bundle -{ - val req_cmd = (new FIFOIO) { new MemReqCmd() } - val req_data = (new FIFOIO) { new MemData() } - val resp = (new PipeIO) { new MemResp() }.flip -} - -class TrackerProbeData extends Bundle { - val tile_id = Bits(width = TILE_ID_BITS) -} - -class TrackerAllocReq extends Bundle { - val xact_init = new TransactionInit() - val tile_id = Bits(width = TILE_ID_BITS) -} - -class TrackerDependency extends Bundle { - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) -} - -class ioTileLink extends Bundle { - val xact_init = (new FIFOIO) { new TransactionInit } - val xact_init_data = (new FIFOIO) { new TransactionInitData } - val xact_abort = (new FIFOIO) { new TransactionAbort }.flip - val probe_req = (new FIFOIO) { new ProbeRequest }.flip - val probe_rep = (new FIFOIO) { new ProbeReply } - val probe_rep_data = (new FIFOIO) { new ProbeReplyData } - val xact_rep = (new FIFOIO) { new TransactionReply }.flip - val xact_finish = (new FIFOIO) { new TransactionFinish } - val incoherent = Bool(OUTPUT) -} - -class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { - val io = new Bundle { - val alloc_req = (new FIFOIO) { new TrackerAllocReq }.flip - val p_data = (new PipeIO) { new TrackerProbeData }.flip - val can_alloc = Bool(INPUT) - val xact_finish = Bool(INPUT) - val p_rep_cnt_dec = Bits(INPUT, ntiles) - val p_req_cnt_inc = Bits(INPUT, ntiles) - val tile_incoherent = Bits(INPUT, ntiles) - val p_rep_data = (new PipeIO) { new ProbeReplyData }.flip - val x_init_data = (new PipeIO) { new TransactionInitData }.flip - val sent_x_rep_ack = Bool(INPUT) - val p_rep_data_dep = (new PipeIO) { new TrackerDependency }.flip - val x_init_data_dep = (new PipeIO) { new TrackerDependency }.flip - - val mem_req_cmd = (new FIFOIO) { new MemReqCmd } - val mem_req_data = (new FIFOIO) { new MemData } - val mem_req_lock = Bool(OUTPUT) - val probe_req = (new FIFOIO) { new ProbeRequest } - val busy = Bool(OUTPUT) - val addr = Bits(OUTPUT, PADDR_BITS - OFFSET_BITS) - val init_tile_id = Bits(OUTPUT, TILE_ID_BITS) - val p_rep_tile_id = Bits(OUTPUT, TILE_ID_BITS) - val tile_xact_id = Bits(OUTPUT, TILE_XACT_ID_BITS) - val sharer_count = Bits(OUTPUT, TILE_ID_BITS+1) - val x_type = Bits(OUTPUT, X_INIT_TYPE_MAX_BITS) - val push_p_req = Bits(OUTPUT, ntiles) - val pop_p_rep = Bits(OUTPUT, ntiles) - val pop_p_rep_data = Bits(OUTPUT, ntiles) - val pop_p_rep_dep = Bits(OUTPUT, ntiles) - val pop_x_init = Bits(OUTPUT, ntiles) - val pop_x_init_data = Bits(OUTPUT, ntiles) - val pop_x_init_dep = Bits(OUTPUT, ntiles) - val send_x_rep_ack = Bool(OUTPUT) - } - - def doMemReqWrite(req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: PipeIO[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { - req_cmd.bits.rw := Bool(true) - req_data.bits := data.bits - when(req_cmd.ready && req_cmd.valid) { - cmd_sent := Bool(true) - } - when (at_front_of_dep_queue) { - req_cmd.valid := !cmd_sent && req_data.ready && data.valid - lock := data.valid || cmd_sent - when (req_cmd.ready || cmd_sent) { - req_data.valid := data.valid - when(req_data.ready) { - pop_data := UFix(1) << tile_id - when (data.valid) { - mem_cnt := mem_cnt_next - when(mem_cnt === UFix(REFILL_CYCLES-1)) { - pop_dep := UFix(1) << tile_id - trigger := Bool(false) - } - } - } - } - } - } - - def doMemReqRead(req_cmd: FIFOIO[MemReqCmd], trigger: Bool) { - req_cmd.valid := Bool(true) - req_cmd.bits.rw := Bool(false) - when(req_cmd.ready) { - trigger := Bool(false) - } - } - - val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } - val state = Reg(resetVal = s_idle) - val addr_ = Reg{ UFix() } - val x_type_ = Reg{ Bits() } - val init_tile_id_ = Reg{ Bits() } - val tile_xact_id_ = Reg{ Bits() } - val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(ntiles))) - val p_req_flags = Reg(resetVal = Bits(0, width = ntiles)) - val p_rep_tile_id_ = Reg{ Bits() } - val x_needs_read = Reg(resetVal = Bool(false)) - val x_init_data_needs_write = Reg(resetVal = Bool(false)) - val p_rep_data_needs_write = Reg(resetVal = Bool(false)) - val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) - val mem_cnt_next = mem_cnt + UFix(1) - val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) - val p_req_initial_flags = Bits(width = ntiles) - p_req_initial_flags := (if (ntiles == 1) Bits(0) else ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id(log2Up(ntiles)-1,0)))) //TODO: Broadcast only - - io.busy := state != s_idle - io.addr := addr_ - io.init_tile_id := init_tile_id_ - io.p_rep_tile_id := p_rep_tile_id_ - io.tile_xact_id := tile_xact_id_ - io.sharer_count := UFix(ntiles) // TODO: Broadcast only - io.x_type := x_type_ - - io.mem_req_cmd.valid := Bool(false) - io.mem_req_cmd.bits.rw := Bool(false) - io.mem_req_cmd.bits.addr := addr_ - io.mem_req_cmd.bits.tag := UFix(id) - io.mem_req_data.valid := Bool(false) - io.mem_req_data.bits.data := UFix(0) - io.mem_req_lock := Bool(false) - io.probe_req.valid := Bool(false) - io.probe_req.bits.p_type := co.getProbeRequestType(x_type_, UFix(0)) - io.probe_req.bits.global_xact_id := UFix(id) - io.probe_req.bits.addr := addr_ - io.push_p_req := Bits(0, width = ntiles) - io.pop_p_rep := Bits(0, width = ntiles) - io.pop_p_rep_data := Bits(0, width = ntiles) - io.pop_p_rep_dep := Bits(0, width = ntiles) - io.pop_x_init := Bits(0, width = ntiles) - io.pop_x_init_data := Bits(0, width = ntiles) - io.pop_x_init_dep := Bits(0, width = ntiles) - io.send_x_rep_ack := Bool(false) - - switch (state) { - is(s_idle) { - when( io.alloc_req.valid && io.can_alloc ) { - addr_ := io.alloc_req.bits.xact_init.addr - x_type_ := io.alloc_req.bits.xact_init.x_type - init_tile_id_ := io.alloc_req.bits.tile_id - tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id - x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) - x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) - p_req_flags := p_req_initial_flags - mem_cnt := UFix(0) - p_w_mem_cmd_sent := Bool(false) - x_w_mem_cmd_sent := Bool(false) - io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id - if(ntiles > 1) { - p_rep_count := PopCount(p_req_initial_flags) - state := Mux(p_req_initial_flags.orR, s_probe, s_mem) - } else state := s_mem - } - } - is(s_probe) { - when(p_req_flags.orR) { - io.push_p_req := p_req_flags - io.probe_req.valid := Bool(true) - } - when(io.p_req_cnt_inc.orR) { - p_req_flags := p_req_flags & ~io.p_req_cnt_inc // unflag sent reqs - } - when(io.p_rep_cnt_dec.orR) { - val dec = PopCount(io.p_rep_cnt_dec) - io.pop_p_rep := io.p_rep_cnt_dec - if(ntiles > 1) p_rep_count := p_rep_count - dec - when(p_rep_count === dec) { - state := s_mem - } - } - when(io.p_data.valid) { - p_rep_data_needs_write := Bool(true) - p_rep_tile_id_ := io.p_data.bits.tile_id - } - } - is(s_mem) { - when (p_rep_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, - io.mem_req_data, - io.mem_req_lock, - io.p_rep_data, - p_rep_data_needs_write, - p_w_mem_cmd_sent, - io.pop_p_rep_data, - io.pop_p_rep_dep, - io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id)), - p_rep_tile_id_) - } . elsewhen(x_init_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, - io.mem_req_data, - io.mem_req_lock, - io.x_init_data, - x_init_data_needs_write, - x_w_mem_cmd_sent, - io.pop_x_init_data, - io.pop_x_init_dep, - io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id)), - init_tile_id_) - } . elsewhen (x_needs_read) { - doMemReqRead(io.mem_req_cmd, x_needs_read) - } . otherwise { - state := Mux(co.needsAckReply(x_type_, UFix(0)), s_ack, s_busy) - } - } - is(s_ack) { - io.send_x_rep_ack := Bool(true) - when(io.sent_x_rep_ack) { state := s_busy } - } - is(s_busy) { // Nothing left to do but wait for transaction to complete - when (io.xact_finish) { - state := s_idle - } - } - } -} - -abstract class CoherenceHub(ntiles: Int, co: CoherencePolicy) extends Component { - val io = new Bundle { - val tiles = Vec(ntiles) { new ioTileLink() }.flip - val mem = new ioMem - } -} - -class CoherenceHubNull(co: ThreeStateIncoherence) extends CoherenceHub(1, co) -{ - val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.x_type === co.xactInitWriteback - x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp - io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) - io.mem.req_cmd.bits.rw := is_write - io.mem.req_cmd.bits.tag := x_init.bits.tile_xact_id - io.mem.req_cmd.bits.addr := x_init.bits.addr - io.mem.req_data <> io.tiles(0).xact_init_data - - val x_rep = io.tiles(0).xact_rep - x_rep.bits.x_type := Mux(io.mem.resp.valid, co.xactReplyData, co.xactReplyAck) - x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) - x_rep.bits.global_xact_id := UFix(0) // don't care - x_rep.bits.data := io.mem.resp.bits.data - x_rep.bits.require_ack := Bool(true) - x_rep.valid := io.mem.resp.valid || x_init.valid && is_write && io.mem.req_cmd.ready - - io.tiles(0).xact_abort.valid := Bool(false) - io.tiles(0).xact_finish.ready := Bool(true) - io.tiles(0).probe_req.valid := Bool(false) - io.tiles(0).probe_rep.ready := Bool(true) - io.tiles(0).probe_rep_data.ready := Bool(true) -} - - -class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceHub(ntiles, co) -{ - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _, co)) - - val busy_arr = Vec(NGLOBAL_XACTS){ Bool() } - val addr_arr = Vec(NGLOBAL_XACTS){ Bits(width=PADDR_BITS-OFFSET_BITS) } - val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_ID_BITS) } - val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_XACT_ID_BITS) } - val x_type_arr = Vec(NGLOBAL_XACTS){ Bits(width=X_INIT_TYPE_MAX_BITS) } - val sh_count_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_ID_BITS) } - val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } - - val do_free_arr = Vec(NGLOBAL_XACTS){ Bool() } - val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Bool()} } - val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Bool()} } - val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } - val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width = TILE_ID_BITS) } - val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } - - for( i <- 0 until NGLOBAL_XACTS) { - val t = trackerList(i).io - busy_arr(i) := t.busy - addr_arr(i) := t.addr - init_tile_id_arr(i) := t.init_tile_id - tile_xact_id_arr(i) := t.tile_xact_id - x_type_arr(i) := t.x_type - sh_count_arr(i) := t.sharer_count - send_x_rep_ack_arr(i) := t.send_x_rep_ack - t.xact_finish := do_free_arr(i) - t.p_data.bits.tile_id := p_data_tile_id_arr(i) - t.p_data.valid := p_data_valid_arr(i) - t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits - t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits - t.tile_incoherent := (Vec(io.tiles.map(_.incoherent)) { Bool() }).toBits - t.sent_x_rep_ack := sent_x_rep_ack_arr(i) - do_free_arr(i) := Bool(false) - sent_x_rep_ack_arr(i) := Bool(false) - p_data_tile_id_arr(i) := Bits(0, width = TILE_ID_BITS) - p_data_valid_arr(i) := Bool(false) - for( j <- 0 until ntiles) { - p_rep_cnt_dec_arr(i)(j) := Bool(false) - p_req_cnt_inc_arr(i)(j) := Bool(false) - } - } - - val p_rep_data_dep_list = List.fill(ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY - val x_init_data_dep_list = List.fill(ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY - - // Free finished transactions - for( j <- 0 until ntiles ) { - val finish = io.tiles(j).xact_finish - when (finish.valid) { - do_free_arr(finish.bits.global_xact_id) := Bool(true) - } - finish.ready := Bool(true) - } - - // Reply to initial requestor - // Forward memory responses from mem to tile or arbitrate to ack - val mem_idx = io.mem.resp.bits.tag - val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) - for( j <- 0 until ntiles ) { - val rep = io.tiles(j).xact_rep - rep.bits.x_type := UFix(0) - rep.bits.tile_xact_id := UFix(0) - rep.bits.global_xact_id := UFix(0) - rep.bits.data := io.mem.resp.bits.data - rep.bits.require_ack := Bool(true) - rep.valid := Bool(false) - when(io.mem.resp.valid && (UFix(j) === init_tile_id_arr(mem_idx))) { - rep.bits.x_type := co.getTransactionReplyType(x_type_arr(mem_idx), sh_count_arr(mem_idx)) - rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) - rep.bits.global_xact_id := mem_idx - rep.valid := Bool(true) - } . otherwise { - rep.bits.x_type := co.getTransactionReplyType(x_type_arr(ack_idx), sh_count_arr(ack_idx)) - rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) - rep.bits.global_xact_id := ack_idx - when (UFix(j) === init_tile_id_arr(ack_idx)) { - rep.valid := send_x_rep_ack_arr.toBits.orR - sent_x_rep_ack_arr(ack_idx) := rep.ready - } - } - } - io.mem.resp.ready := io.tiles(init_tile_id_arr(mem_idx)).xact_rep.ready - - // Create an arbiter for the one memory port - // We have to arbitrate between the different trackers' memory requests - // and once we have picked a request, get the right write data - val mem_req_cmd_arb = (new Arbiter(NGLOBAL_XACTS)) { new MemReqCmd() } - val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemData() } - for( i <- 0 until NGLOBAL_XACTS ) { - mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd - mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data - mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock - } - io.mem.req_cmd <> Queue(mem_req_cmd_arb.io.out) - io.mem.req_data <> Queue(mem_req_data_arb.io.out) - - // Handle probe replies, which may or may not have data - for( j <- 0 until ntiles ) { - val p_rep = io.tiles(j).probe_rep - val p_rep_data = io.tiles(j).probe_rep_data - val idx = p_rep.bits.global_xact_id - val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) - val do_pop = foldR(pop_p_reps)(_ || _) - p_rep.ready := Bool(true) - p_rep_data_dep_list(j).io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits) - p_rep_data_dep_list(j).io.enq.bits.global_xact_id := p_rep.bits.global_xact_id - p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) - when (p_rep.valid && co.messageHasData(p_rep.bits)) { - p_data_valid_arr(idx) := Bool(true) - p_data_tile_id_arr(idx) := UFix(j) - } - p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) - } - for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid - trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits - - trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) - trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) - - for( j <- 0 until ntiles) { - val p_rep = io.tiles(j).probe_rep - p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) - } - } - - // Nack conflicting transaction init attempts - val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } - val abort_state_arr = Vec(ntiles) { Reg(resetVal = s_idle) } - val want_to_abort_arr = Vec(ntiles) { Bool() } - for( j <- 0 until ntiles ) { - val x_init = io.tiles(j).xact_init - val x_init_data = io.tiles(j).xact_init_data - val x_abort = io.tiles(j).xact_abort - val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) - val conflicts = Vec(NGLOBAL_XACTS) { Bool() } - for( i <- 0 until NGLOBAL_XACTS) { - val t = trackerList(i).io - conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.addr) - } - x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id - want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits))) - - x_abort.valid := Bool(false) - switch(abort_state_arr(j)) { - is(s_idle) { - when(want_to_abort_arr(j)) { - when(co.messageHasData(x_init.bits)) { - abort_state_arr(j) := s_abort_drain - } . otherwise { - abort_state_arr(j) := s_abort_send - } - } - } - is(s_abort_drain) { // raises x_init_data.ready below - when(x_init_data.valid) { - abort_cnt := abort_cnt + UFix(1) - when(abort_cnt === ~UFix(0, width = log2Up(REFILL_CYCLES))) { - abort_state_arr(j) := s_abort_send - } - } - } - is(s_abort_send) { // nothing is dequeued for now - x_abort.valid := Bool(true) - when(x_abort.ready) { // raises x_init.ready below - abort_state_arr(j) := s_idle - } - } - } - } - - // Handle transaction initiation requests - // Only one allocation per cycle - // Init requests may or may not have data - val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } - val init_arb = (new Arbiter(ntiles)) { new TrackerAllocReq() } - for( i <- 0 until NGLOBAL_XACTS ) { - alloc_arb.io.in(i).valid := !trackerList(i).io.busy - trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready - trackerList(i).io.alloc_req.bits := init_arb.io.out.bits - trackerList(i).io.alloc_req.valid := init_arb.io.out.valid - - trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits - trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid - trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) - trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) - } - for( j <- 0 until ntiles ) { - val x_init = io.tiles(j).xact_init - val x_init_data = io.tiles(j).xact_init_data - val x_init_data_dep = x_init_data_dep_list(j).io.deq - val x_abort = io.tiles(j).xact_abort - init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid - init_arb.io.in(j).bits.xact_init := x_init.bits - init_arb.io.in(j).bits.tile_id := UFix(j) - val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) - val do_pop = foldR(pop_x_inits)(_||_) - x_init_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(x_init.bits) && (abort_state_arr(j) === s_idle) - x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) - x_init.ready := (x_abort.valid && x_abort.ready) || do_pop - x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) - x_init_data_dep.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) - } - - alloc_arb.io.out.ready := init_arb.io.out.valid - - // Handle probe request generation - // Must arbitrate for each request port - val p_req_arb_arr = List.fill(ntiles)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) - for( j <- 0 until ntiles ) { - for( i <- 0 until NGLOBAL_XACTS ) { - val t = trackerList(i).io - p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits - p_req_arb_arr(j).io.in(i).valid := t.probe_req.valid && t.push_p_req(j) - p_req_cnt_inc_arr(i)(j) := p_req_arb_arr(j).io.in(i).ready - } - p_req_arb_arr(j).io.out <> io.tiles(j).probe_req - } - -} From 916c1019af027e7d42133676644f70c4cbb9df06 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Tue, 9 Oct 2012 13:02:58 -0700 Subject: [PATCH 132/374] fixed memdessert unpacking --- uncore/memserdes.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/memserdes.scala b/uncore/memserdes.scala index 712dec16..e8d99514 100644 --- a/uncore/memserdes.scala +++ b/uncore/memserdes.scala @@ -144,8 +144,8 @@ class MemDessert extends Component // test rig side val req_cmd = in_buf >> UFix(((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH - (abits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH)*MEM_BACKUP_WIDTH) io.wide.req_cmd.valid := state === s_cmd io.wide.req_cmd.bits.tag := req_cmd - io.wide.req_cmd.bits.addr := req_cmd.toUFix >> UFix(io.wide.req_cmd.bits.tag.width) - io.wide.req_cmd.bits.rw := req_cmd(io.wide.req_cmd.bits.tag.width + io.wide.req_cmd.bits.addr.width) + io.wide.req_cmd.bits.addr := req_cmd.toUFix >> UFix(io.wide.req_cmd.bits.tag.width + io.wide.req_cmd.bits.rw.width) + io.wide.req_cmd.bits.rw := req_cmd(io.wide.req_cmd.bits.tag.width) io.wide.req_data.valid := state === s_data io.wide.req_data.bits.data := in_buf >> UFix(((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH - (dbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH)*MEM_BACKUP_WIDTH) From 3973aef9388edfffb1910ea21c7a2e45723be7ab Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 9 Oct 2012 18:04:55 -0700 Subject: [PATCH 133/374] handle structural hazard on LLC tags --- uncore/src/llc.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index 2bb2b62a..cb3fbb6a 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -361,10 +361,10 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val stall_s1 = Bool() val replay_s1 = Reg(resetVal = Bool(false)) - val s1_valid = Reg(io.cpu.req_cmd.valid && !stall_s1 || replay_s1, resetVal = Bool(false)) + val s1_valid = Reg(io.cpu.req_cmd.fire() || replay_s1, resetVal = Bool(false)) replay_s1 := s1_valid && stall_s1 val s1 = Reg() { new MemReqCmd } - when (io.cpu.req_cmd.valid && io.cpu.req_cmd.ready) { s1 := io.cpu.req_cmd.bits } + when (io.cpu.req_cmd.fire()) { s1 := io.cpu.req_cmd.bits } val stall_s2 = Bool() val s2_valid = Reg(resetVal = Bool(false)) @@ -430,7 +430,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da stall_s2 := s2_valid && !(dataArb.io.in(1).ready && writeback.io.req(0).ready && mshr.io.cpu.ready) io.cpu.resp <> data.io.resp - io.cpu.req_cmd.ready := !stall_s1 && !replay_s1 + io.cpu.req_cmd.ready := !stall_s1 && !replay_s1 && !tag_we io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready io.mem.req_cmd <> memCmdArb.io.out io.mem.req_data <> writeback.io.mem.req_data From 9610622ab0500addf535474ef91f3de518e5b947 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Wed, 10 Oct 2012 12:41:11 -0700 Subject: [PATCH 134/374] moving memserdes + slowio into src --- uncore/{ => src}/memserdes.scala | 0 uncore/{ => src}/slowio.scala | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename uncore/{ => src}/memserdes.scala (100%) rename uncore/{ => src}/slowio.scala (100%) diff --git a/uncore/memserdes.scala b/uncore/src/memserdes.scala similarity index 100% rename from uncore/memserdes.scala rename to uncore/src/memserdes.scala diff --git a/uncore/slowio.scala b/uncore/src/slowio.scala similarity index 100% rename from uncore/slowio.scala rename to uncore/src/slowio.scala From 08ab076217312efd98bf697964e7db7573edfc94 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Wed, 10 Oct 2012 15:42:39 -0700 Subject: [PATCH 135/374] forgot to change package + using fromBits in memserdes instead of manual unpacking --- uncore/src/consts.scala | 3 +++ uncore/src/memserdes.scala | 6 ++---- uncore/src/slowio.scala | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index 95b07cc8..a1e87f41 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -10,6 +10,9 @@ object Constants val HAVE_FPU = true val HAVE_VEC = true + val HTIF_WIDTH = 16 + val MEM_BACKUP_WIDTH = HTIF_WIDTH + val M_X = Bits("b????", 4); val M_XRD = Bits("b0000", 4); // int load val M_XWR = Bits("b0001", 4); // int store diff --git a/uncore/src/memserdes.scala b/uncore/src/memserdes.scala index e8d99514..c44e2543 100644 --- a/uncore/src/memserdes.scala +++ b/uncore/src/memserdes.scala @@ -1,4 +1,4 @@ -package rocket +package uncore import Chisel._ import Node._ @@ -143,9 +143,7 @@ class MemDessert extends Component // test rig side val req_cmd = in_buf >> UFix(((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH - (abits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH)*MEM_BACKUP_WIDTH) io.wide.req_cmd.valid := state === s_cmd - io.wide.req_cmd.bits.tag := req_cmd - io.wide.req_cmd.bits.addr := req_cmd.toUFix >> UFix(io.wide.req_cmd.bits.tag.width + io.wide.req_cmd.bits.rw.width) - io.wide.req_cmd.bits.rw := req_cmd(io.wide.req_cmd.bits.tag.width) + io.wide.req_cmd.bits := io.wide.req_cmd.bits.fromBits(req_cmd) io.wide.req_data.valid := state === s_data io.wide.req_data.bits.data := in_buf >> UFix(((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH - (dbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH)*MEM_BACKUP_WIDTH) diff --git a/uncore/src/slowio.scala b/uncore/src/slowio.scala index 068e90c5..fa6fef94 100644 --- a/uncore/src/slowio.scala +++ b/uncore/src/slowio.scala @@ -1,4 +1,4 @@ -package rocket +package uncore import Chisel._ import Constants._ From 1418604bf01252c9b88bf337288b0a303ccf8717 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 15 Oct 2012 18:52:48 -0700 Subject: [PATCH 136/374] new constants organization --- uncore/src/consts.scala | 103 +++++++++++++++++++-------------------- uncore/src/package.scala | 10 ++++ 2 files changed, 61 insertions(+), 52 deletions(-) create mode 100644 uncore/src/package.scala diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index a1e87f41..098bc1d6 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -1,17 +1,49 @@ package uncore +package constants import Chisel._ -import scala.math._ -object Constants +abstract trait MulticoreConstants { + val NTILES: Int + val TILE_ID_BITS = log2Up(NTILES)+1 +} + +abstract trait CoherenceConfigConstants { + val ENABLE_SHARING: Boolean + val ENABLE_CLEAN_EXCLUSIVE: Boolean +} + +trait UncoreConstants { + val NGLOBAL_XACTS = 8 + val GLOBAL_XACT_ID_BITS = log2Up(NGLOBAL_XACTS) + val CACHE_DATA_SIZE_IN_BYTES = 1 << 6 +} + +trait TileLinkTypeConstants { + val X_INIT_TYPE_MAX_BITS = 2 + val X_REP_TYPE_MAX_BITS = 3 + val P_REQ_TYPE_MAX_BITS = 2 + val P_REP_TYPE_MAX_BITS = 3 +} + +trait TileLinkSizeConstants extends + TileLinkTypeConstants { - val NTILES = 1 - val HAVE_RVC = false - val HAVE_FPU = true - val HAVE_VEC = true + val TILE_XACT_ID_BITS = 5 + val X_INIT_WRITE_MASK_BITS = 6 + val X_INIT_SUBWORD_ADDR_BITS = 3 + val X_INIT_ATOMIC_OP_BITS = 4 +} - val HTIF_WIDTH = 16 - val MEM_BACKUP_WIDTH = HTIF_WIDTH +trait MemoryOpConstants { + val MT_X = Bits("b???", 3); + val MT_B = Bits("b000", 3); + val MT_H = Bits("b001", 3); + val MT_W = Bits("b010", 3); + val MT_D = Bits("b011", 3); + val MT_BU = Bits("b100", 3); + val MT_HU = Bits("b101", 3); + val MT_WU = Bits("b110", 3); val M_X = Bits("b????", 4); val M_XRD = Bits("b0000", 4); // int load @@ -30,52 +62,19 @@ object Constants val M_XA_MAX = Bits("b1101", 4); val M_XA_MINU = Bits("b1110", 4); val M_XA_MAXU = Bits("b1111", 4); - - val PADDR_BITS = 40; - val VADDR_BITS = 43; - val PGIDX_BITS = 13; - val PPN_BITS = PADDR_BITS-PGIDX_BITS; - val VPN_BITS = VADDR_BITS-PGIDX_BITS; - val ASID_BITS = 7; - val PERM_BITS = 6; +} - // rocketNBDCache parameters - val DCACHE_PORTS = 3 - val CPU_DATA_BITS = 64; - val CPU_TAG_BITS = 9; - val DCACHE_TAG_BITS = log2Up(DCACHE_PORTS) + CPU_TAG_BITS - val OFFSET_BITS = 6; // log2(cache line size in bytes) - val NMSHR = if (HAVE_VEC) 4 else 2 // number of primary misses - val NRPQ = 16; // number of secondary misses - val NSDQ = 17; // number of secondary stores/AMOs - val LG_REFILL_WIDTH = 4; // log2(cache bus width in bytes) - val IDX_BITS = 7; - val TAG_BITS = PADDR_BITS - OFFSET_BITS - IDX_BITS; - val NWAYS = 4 - require(IDX_BITS+OFFSET_BITS <= PGIDX_BITS); +trait HTIFConstants { + val HTIF_WIDTH = 16 +} - // coherence parameters - val ENABLE_SHARING = true - val ENABLE_CLEAN_EXCLUSIVE = true - - - val COHERENCE_DATA_BITS = (1 << OFFSET_BITS)*8 - val TILE_ID_BITS = log2Up(NTILES)+1 - val TILE_XACT_ID_BITS = log2Up(NMSHR)+3 - val NGLOBAL_XACTS = 8 - val GLOBAL_XACT_ID_BITS = log2Up(NGLOBAL_XACTS) - - val X_INIT_TYPE_MAX_BITS = 2 - val X_INIT_WRITE_MASK_BITS = OFFSET_BITS - val X_INIT_SUBWORD_ADDR_BITS = log2Up(OFFSET_BITS) - val X_INIT_ATOMIC_OP_BITS = 4 - val X_REP_TYPE_MAX_BITS = 3 - val P_REQ_TYPE_MAX_BITS = 2 - val P_REP_TYPE_MAX_BITS = 3 - - // external memory interface +trait MemoryInterfaceConstants extends + HTIFConstants with + UncoreConstants with + TileLinkSizeConstants +{ val MEM_TAG_BITS = max(TILE_XACT_ID_BITS, GLOBAL_XACT_ID_BITS) val MEM_DATA_BITS = 128 - val REFILL_CYCLES = (1 << OFFSET_BITS)*8/MEM_DATA_BITS - + val REFILL_CYCLES = CACHE_DATA_SIZE_IN_BYTES*8/MEM_DATA_BITS + val MEM_BACKUP_WIDTH = HTIF_WIDTH } diff --git a/uncore/src/package.scala b/uncore/src/package.scala new file mode 100644 index 00000000..17954b54 --- /dev/null +++ b/uncore/src/package.scala @@ -0,0 +1,10 @@ +package uncore +import uncore.constants._ + +object Constants extends + MemoryOpConstants with + MemoryInterfaceConstants +{ + +} + From 8509cda8131b896540b1ddf33c879298c42b5870 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 16 Oct 2012 13:58:18 -0700 Subject: [PATCH 137/374] Refined traits for use with rocket asserts, added UncoreConfiguration to handle ntiles --- uncore/src/consts.scala | 21 +++++-- uncore/src/package.scala | 7 ++- uncore/src/uncore.scala | 125 ++++++++++++++++++++------------------- 3 files changed, 85 insertions(+), 68 deletions(-) diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index 098bc1d6..b7a70bac 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -2,11 +2,7 @@ package uncore package constants import Chisel._ - -abstract trait MulticoreConstants { - val NTILES: Int - val TILE_ID_BITS = log2Up(NTILES)+1 -} +import scala.math.max abstract trait CoherenceConfigConstants { val ENABLE_SHARING: Boolean @@ -19,6 +15,10 @@ trait UncoreConstants { val CACHE_DATA_SIZE_IN_BYTES = 1 << 6 } +trait CacheConstants extends UncoreConstants { + val OFFSET_BITS = log2Up(CACHE_DATA_SIZE_IN_BYTES) +} + trait TileLinkTypeConstants { val X_INIT_TYPE_MAX_BITS = 2 val X_REP_TYPE_MAX_BITS = 3 @@ -78,3 +78,14 @@ trait MemoryInterfaceConstants extends val REFILL_CYCLES = CACHE_DATA_SIZE_IN_BYTES*8/MEM_DATA_BITS val MEM_BACKUP_WIDTH = HTIF_WIDTH } + +trait AddressConstants { + val PADDR_BITS = 40; + val VADDR_BITS = 43; + val PGIDX_BITS = 13; + val PPN_BITS = PADDR_BITS-PGIDX_BITS; + val VPN_BITS = VADDR_BITS-PGIDX_BITS; + val ASID_BITS = 7; + val PERM_BITS = 6; +} + diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 17954b54..e3a8f554 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -1,10 +1,13 @@ package uncore import uncore.constants._ +//TODO: When compiler bug SI-5604 is fixed in 2.10, change object Constants to +// package object uncore and remove import Constants._'s from other files object Constants extends MemoryOpConstants with - MemoryInterfaceConstants + MemoryInterfaceConstants with + CacheConstants with + AddressConstants { - } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 1220a56f..dcd1a819 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -36,13 +36,14 @@ class ioMemPipe() extends Bundle val resp = (new PipeIO) { new MemResp() }.flip } -class TrackerProbeData extends Bundle { - val tile_id = Bits(width = TILE_ID_BITS) +class TrackerProbeData(implicit conf: UncoreConfiguration) extends Bundle { + val tile_id = Bits(width = conf.tile_id_bits) } -class TrackerAllocReq extends Bundle { +class TrackerAllocReq(implicit conf: UncoreConfiguration) extends Bundle { val xact_init = new TransactionInit() - val tile_id = Bits(width = TILE_ID_BITS) + val tile_id = Bits(width = conf.tile_id_bits) + override def clone = { new TrackerAllocReq().asInstanceOf[this.type] } } class TrackerDependency extends Bundle { @@ -61,15 +62,15 @@ class ioTileLink extends Bundle { val incoherent = Bool(OUTPUT) } -class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { +class XactTracker(id: Int, co: CoherencePolicy)(implicit conf: UncoreConfiguration) extends Component { val io = new Bundle { val alloc_req = (new FIFOIO) { new TrackerAllocReq }.flip val p_data = (new PipeIO) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) - val p_rep_cnt_dec = Bits(INPUT, ntiles) - val p_req_cnt_inc = Bits(INPUT, ntiles) - val tile_incoherent = Bits(INPUT, ntiles) + val p_rep_cnt_dec = Bits(INPUT, conf.ntiles) + val p_req_cnt_inc = Bits(INPUT, conf.ntiles) + val tile_incoherent = Bits(INPUT, conf.ntiles) val p_rep_data = (new PipeIO) { new ProbeReplyData }.flip val x_init_data = (new PipeIO) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) @@ -82,18 +83,18 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val probe_req = (new FIFOIO) { new ProbeRequest } val busy = Bool(OUTPUT) val addr = Bits(OUTPUT, PADDR_BITS - OFFSET_BITS) - val init_tile_id = Bits(OUTPUT, TILE_ID_BITS) - val p_rep_tile_id = Bits(OUTPUT, TILE_ID_BITS) + val init_tile_id = Bits(OUTPUT, conf.tile_id_bits) + val p_rep_tile_id = Bits(OUTPUT, conf.tile_id_bits) val tile_xact_id = Bits(OUTPUT, TILE_XACT_ID_BITS) - val sharer_count = Bits(OUTPUT, TILE_ID_BITS+1) + val sharer_count = Bits(OUTPUT, conf.tile_id_bits+1) val x_type = Bits(OUTPUT, X_INIT_TYPE_MAX_BITS) - val push_p_req = Bits(OUTPUT, ntiles) - val pop_p_rep = Bits(OUTPUT, ntiles) - val pop_p_rep_data = Bits(OUTPUT, ntiles) - val pop_p_rep_dep = Bits(OUTPUT, ntiles) - val pop_x_init = Bits(OUTPUT, ntiles) - val pop_x_init_data = Bits(OUTPUT, ntiles) - val pop_x_init_dep = Bits(OUTPUT, ntiles) + val push_p_req = Bits(OUTPUT, conf.ntiles) + val pop_p_rep = Bits(OUTPUT, conf.ntiles) + val pop_p_rep_data = Bits(OUTPUT, conf.ntiles) + val pop_p_rep_dep = Bits(OUTPUT, conf.ntiles) + val pop_x_init = Bits(OUTPUT, conf.ntiles) + val pop_x_init_data = Bits(OUTPUT, conf.ntiles) + val pop_x_init_dep = Bits(OUTPUT, conf.ntiles) val send_x_rep_ack = Bool(OUTPUT) } @@ -136,8 +137,8 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val x_type_ = Reg{ Bits() } val init_tile_id_ = Reg{ Bits() } val tile_xact_id_ = Reg{ Bits() } - val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(ntiles))) - val p_req_flags = Reg(resetVal = Bits(0, width = ntiles)) + val p_rep_count = if (conf.ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ntiles))) + val p_req_flags = Reg(resetVal = Bits(0, width = conf.ntiles)) val p_rep_tile_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) val x_init_data_needs_write = Reg(resetVal = Bool(false)) @@ -147,15 +148,15 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) - val p_req_initial_flags = Bits(width = ntiles) - p_req_initial_flags := (if (ntiles == 1) Bits(0) else ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id(log2Up(ntiles)-1,0)))) //TODO: Broadcast only + val p_req_initial_flags = Bits(width = conf.ntiles) + p_req_initial_flags := (if (conf.ntiles == 1) Bits(0) else ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id(log2Up(conf.ntiles)-1,0)))) //TODO: Broadcast only io.busy := state != s_idle io.addr := addr_ io.init_tile_id := init_tile_id_ io.p_rep_tile_id := p_rep_tile_id_ io.tile_xact_id := tile_xact_id_ - io.sharer_count := UFix(ntiles) // TODO: Broadcast only + io.sharer_count := UFix(conf.ntiles) // TODO: Broadcast only io.x_type := x_type_ io.mem_req_cmd.valid := Bool(false) @@ -169,13 +170,13 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { io.probe_req.bits.p_type := co.getProbeRequestType(x_type_, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) io.probe_req.bits.addr := addr_ - io.push_p_req := Bits(0, width = ntiles) - io.pop_p_rep := Bits(0, width = ntiles) - io.pop_p_rep_data := Bits(0, width = ntiles) - io.pop_p_rep_dep := Bits(0, width = ntiles) - io.pop_x_init := Bits(0, width = ntiles) - io.pop_x_init_data := Bits(0, width = ntiles) - io.pop_x_init_dep := Bits(0, width = ntiles) + io.push_p_req := Bits(0, width = conf.ntiles) + io.pop_p_rep := Bits(0, width = conf.ntiles) + io.pop_p_rep_data := Bits(0, width = conf.ntiles) + io.pop_p_rep_dep := Bits(0, width = conf.ntiles) + io.pop_x_init := Bits(0, width = conf.ntiles) + io.pop_x_init_data := Bits(0, width = conf.ntiles) + io.pop_x_init_dep := Bits(0, width = conf.ntiles) io.send_x_rep_ack := Bool(false) switch (state) { @@ -192,7 +193,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id - if(ntiles > 1) { + if(conf.ntiles > 1) { p_rep_count := PopCount(p_req_initial_flags) state := Mux(p_req_initial_flags.orR, s_probe, s_mem) } else state := s_mem @@ -209,7 +210,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { when(io.p_rep_cnt_dec.orR) { val dec = PopCount(io.p_rep_cnt_dec) io.pop_p_rep := io.p_rep_cnt_dec - if(ntiles > 1) p_rep_count := p_rep_count - dec + if(conf.ntiles > 1) p_rep_count := p_rep_count - dec when(p_rep_count === dec) { state := s_mem } @@ -260,14 +261,16 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { } } -abstract class CoherenceHub(ntiles: Int, co: CoherencePolicy) extends Component { +case class UncoreConfiguration(ntiles: Int, tile_id_bits: Int) + +abstract class CoherenceHub(co: CoherencePolicy)(implicit conf: UncoreConfiguration) extends Component { val io = new Bundle { - val tiles = Vec(ntiles) { new ioTileLink() }.flip + val tiles = Vec(conf.ntiles) { new ioTileLink }.flip val mem = new ioMem } } -class CoherenceHubNull(co: ThreeStateIncoherence) extends CoherenceHub(1, co) +class CoherenceHubNull(co: ThreeStateIncoherence)(implicit conf: UncoreConfiguration) extends CoherenceHub(co)(conf) { val x_init = io.tiles(0).xact_init val is_write = x_init.bits.x_type === co.xactInitWriteback @@ -294,23 +297,23 @@ class CoherenceHubNull(co: ThreeStateIncoherence) extends CoherenceHub(1, co) } -class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceHub(ntiles, co) +class CoherenceHubBroadcast(co: CoherencePolicy)(implicit conf: UncoreConfiguration) extends CoherenceHub(co)(conf) { - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _, co)) + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_, co)) val busy_arr = Vec(NGLOBAL_XACTS){ Bool() } val addr_arr = Vec(NGLOBAL_XACTS){ Bits(width=PADDR_BITS-OFFSET_BITS) } - val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_ID_BITS) } + val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.tile_id_bits) } val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_XACT_ID_BITS) } val x_type_arr = Vec(NGLOBAL_XACTS){ Bits(width=X_INIT_TYPE_MAX_BITS) } - val sh_count_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_ID_BITS) } + val sh_count_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.tile_id_bits) } val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } val do_free_arr = Vec(NGLOBAL_XACTS){ Bool() } - val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Bool()} } - val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Bool()} } + val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ntiles){ Bool()} } + val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ntiles){ Bool()} } val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } - val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width = TILE_ID_BITS) } + val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.tile_id_bits) } val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } for( i <- 0 until NGLOBAL_XACTS) { @@ -331,19 +334,19 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH t.sent_x_rep_ack := sent_x_rep_ack_arr(i) do_free_arr(i) := Bool(false) sent_x_rep_ack_arr(i) := Bool(false) - p_data_tile_id_arr(i) := Bits(0, width = TILE_ID_BITS) + p_data_tile_id_arr(i) := Bits(0, width = conf.tile_id_bits) p_data_valid_arr(i) := Bool(false) - for( j <- 0 until ntiles) { + for( j <- 0 until conf.ntiles) { p_rep_cnt_dec_arr(i)(j) := Bool(false) p_req_cnt_inc_arr(i)(j) := Bool(false) } } - val p_rep_data_dep_list = List.fill(ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY - val x_init_data_dep_list = List.fill(ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY + val p_rep_data_dep_list = List.fill(conf.ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY + val x_init_data_dep_list = List.fill(conf.ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY // Free finished transactions - for( j <- 0 until ntiles ) { + for( j <- 0 until conf.ntiles ) { val finish = io.tiles(j).xact_finish when (finish.valid) { do_free_arr(finish.bits.global_xact_id) := Bool(true) @@ -355,7 +358,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH // Forward memory responses from mem to tile or arbitrate to ack val mem_idx = io.mem.resp.bits.tag val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) - for( j <- 0 until ntiles ) { + for( j <- 0 until conf.ntiles ) { val rep = io.tiles(j).xact_rep rep.bits.x_type := UFix(0) rep.bits.tile_xact_id := UFix(0) @@ -394,7 +397,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH io.mem.req_data <> Queue(mem_req_data_arb.io.out) // Handle probe replies, which may or may not have data - for( j <- 0 until ntiles ) { + for( j <- 0 until conf.ntiles ) { val p_rep = io.tiles(j).probe_rep val p_rep_data = io.tiles(j).probe_rep_data val idx = p_rep.bits.global_xact_id @@ -414,10 +417,10 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits - trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) - trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) + trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until conf.ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) + trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until conf.ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) - for( j <- 0 until ntiles) { + for( j <- 0 until conf.ntiles) { val p_rep = io.tiles(j).probe_rep p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) } @@ -425,9 +428,9 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH // Nack conflicting transaction init attempts val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } - val abort_state_arr = Vec(ntiles) { Reg(resetVal = s_idle) } - val want_to_abort_arr = Vec(ntiles) { Bool() } - for( j <- 0 until ntiles ) { + val abort_state_arr = Vec(conf.ntiles) { Reg(resetVal = s_idle) } + val want_to_abort_arr = Vec(conf.ntiles) { Bool() } + for( j <- 0 until conf.ntiles ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_abort = io.tiles(j).xact_abort @@ -472,7 +475,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH // Only one allocation per cycle // Init requests may or may not have data val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } - val init_arb = (new Arbiter(ntiles)) { new TrackerAllocReq() } + val init_arb = (new Arbiter(conf.ntiles)) { new TrackerAllocReq } for( i <- 0 until NGLOBAL_XACTS ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready @@ -481,10 +484,10 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid - trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) - trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) + trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until conf.ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) + trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until conf.ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) } - for( j <- 0 until ntiles ) { + for( j <- 0 until conf.ntiles ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_init_data_dep = x_init_data_dep_list(j).io.deq @@ -505,8 +508,8 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH // Handle probe request generation // Must arbitrate for each request port - val p_req_arb_arr = List.fill(ntiles)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) - for( j <- 0 until ntiles ) { + val p_req_arb_arr = List.fill(conf.ntiles)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) + for( j <- 0 until conf.ntiles ) { for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits From 9df5cfa5524249d8b1ce3f955e5090cd73e79980 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 16 Oct 2012 14:26:33 -0700 Subject: [PATCH 138/374] Factored out tilelink classes --- uncore/src/coherence.scala | 64 --------------------- uncore/src/tilelink.scala | 111 +++++++++++++++++++++++++++++++++++++ uncore/src/uncore.scala | 45 --------------- 3 files changed, 111 insertions(+), 109 deletions(-) create mode 100644 uncore/src/tilelink.scala diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index 7fc03ece..dcf384aa 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -3,70 +3,6 @@ package uncore import Chisel._ import Constants._ -object TransactionInit -{ - def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix) = { - val init = new TransactionInit - init.x_type := x_type - init.addr := addr - init.tile_xact_id := tile_xact_id - init - } - def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix, write_mask: Bits) = { - val init = new TransactionInit - init.x_type := x_type - init.addr := addr - init.tile_xact_id := tile_xact_id - init.write_mask := write_mask - init - } - def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix, subword_addr: UFix, atomic_opcode: UFix) = { - val init = new TransactionInit - init.x_type := x_type - init.addr := addr - init.tile_xact_id := tile_xact_id - init.subword_addr := subword_addr - init.atomic_opcode := atomic_opcode - init - } -} -class TransactionInit extends PhysicalAddress { - val x_type = Bits(width = X_INIT_TYPE_MAX_BITS) - val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) - val write_mask = Bits(width = X_INIT_WRITE_MASK_BITS) - val subword_addr = Bits(width = X_INIT_SUBWORD_ADDR_BITS) - val atomic_opcode = Bits(width = X_INIT_ATOMIC_OP_BITS) -} - -class TransactionInitData extends MemData - -class TransactionAbort extends Bundle { - val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) -} - -class ProbeRequest extends PhysicalAddress { - val p_type = Bits(width = P_REQ_TYPE_MAX_BITS) - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) -} - -class ProbeReply extends Bundle { - val p_type = Bits(width = P_REP_TYPE_MAX_BITS) - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) -} - -class ProbeReplyData extends MemData - -class TransactionReply extends MemData { - val x_type = Bits(width = X_REP_TYPE_MAX_BITS) - val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) - val require_ack = Bool() -} - -class TransactionFinish extends Bundle { - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) -} - object cpuCmdToRW { def apply(cmd: Bits): (Bool, Bool) = { val store = (cmd === M_XWR) diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala new file mode 100644 index 00000000..bddbc0d3 --- /dev/null +++ b/uncore/src/tilelink.scala @@ -0,0 +1,111 @@ +package uncore + +import Chisel._ +import Constants._ + + +class PhysicalAddress extends Bundle { + val addr = UFix(width = PADDR_BITS - OFFSET_BITS) +} + +class MemData extends Bundle { + val data = Bits(width = MEM_DATA_BITS) +} + +class MemReqCmd extends PhysicalAddress { + val rw = Bool() + val tag = Bits(width = MEM_TAG_BITS) +} + +class MemResp extends MemData { + val tag = Bits(width = MEM_TAG_BITS) +} + +class ioMem extends Bundle { + val req_cmd = (new FIFOIO) { new MemReqCmd() } + val req_data = (new FIFOIO) { new MemData() } + val resp = (new FIFOIO) { new MemResp() }.flip +} + +class ioMemPipe extends Bundle { + val req_cmd = (new FIFOIO) { new MemReqCmd() } + val req_data = (new FIFOIO) { new MemData() } + val resp = (new PipeIO) { new MemResp() }.flip +} + +class TransactionInit extends PhysicalAddress { + val x_type = Bits(width = X_INIT_TYPE_MAX_BITS) + val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) + val write_mask = Bits(width = X_INIT_WRITE_MASK_BITS) + val subword_addr = Bits(width = X_INIT_SUBWORD_ADDR_BITS) + val atomic_opcode = Bits(width = X_INIT_ATOMIC_OP_BITS) +} + +object TransactionInit +{ + def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix) = { + val init = new TransactionInit + init.x_type := x_type + init.addr := addr + init.tile_xact_id := tile_xact_id + init + } + def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix, write_mask: Bits) = { + val init = new TransactionInit + init.x_type := x_type + init.addr := addr + init.tile_xact_id := tile_xact_id + init.write_mask := write_mask + init + } + def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix, subword_addr: UFix, atomic_opcode: UFix) = { + val init = new TransactionInit + init.x_type := x_type + init.addr := addr + init.tile_xact_id := tile_xact_id + init.subword_addr := subword_addr + init.atomic_opcode := atomic_opcode + init + } +} + +class TransactionInitData extends MemData + +class TransactionAbort extends Bundle { + val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) +} + +class ProbeRequest extends PhysicalAddress { + val p_type = Bits(width = P_REQ_TYPE_MAX_BITS) + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) +} + +class ProbeReply extends Bundle { + val p_type = Bits(width = P_REP_TYPE_MAX_BITS) + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) +} + +class ProbeReplyData extends MemData + +class TransactionReply extends MemData { + val x_type = Bits(width = X_REP_TYPE_MAX_BITS) + val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) + val require_ack = Bool() +} + +class TransactionFinish extends Bundle { + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) +} + +class ioTileLink extends Bundle { + val xact_init = (new FIFOIO) { new TransactionInit } + val xact_init_data = (new FIFOIO) { new TransactionInitData } + val xact_abort = (new FIFOIO) { new TransactionAbort }.flip + val probe_req = (new FIFOIO) { new ProbeRequest }.flip + val probe_rep = (new FIFOIO) { new ProbeReply } + val probe_rep_data = (new FIFOIO) { new ProbeReplyData } + val xact_rep = (new FIFOIO) { new TransactionReply }.flip + val xact_finish = (new FIFOIO) { new TransactionFinish } + val incoherent = Bool(OUTPUT) +} diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index dcd1a819..ebca0834 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -3,39 +3,6 @@ package uncore import Chisel._ import Constants._ -class PhysicalAddress extends Bundle { - val addr = UFix(width = PADDR_BITS - OFFSET_BITS) -} - -class MemData extends Bundle { - val data = Bits(width = MEM_DATA_BITS) -} - -class MemReqCmd() extends PhysicalAddress -{ - val rw = Bool() - val tag = Bits(width = MEM_TAG_BITS) -} - -class MemResp () extends MemData -{ - val tag = Bits(width = MEM_TAG_BITS) -} - -class ioMem() extends Bundle -{ - val req_cmd = (new FIFOIO) { new MemReqCmd() } - val req_data = (new FIFOIO) { new MemData() } - val resp = (new FIFOIO) { new MemResp() }.flip -} - -class ioMemPipe() extends Bundle -{ - val req_cmd = (new FIFOIO) { new MemReqCmd() } - val req_data = (new FIFOIO) { new MemData() } - val resp = (new PipeIO) { new MemResp() }.flip -} - class TrackerProbeData(implicit conf: UncoreConfiguration) extends Bundle { val tile_id = Bits(width = conf.tile_id_bits) } @@ -50,18 +17,6 @@ class TrackerDependency extends Bundle { val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } -class ioTileLink extends Bundle { - val xact_init = (new FIFOIO) { new TransactionInit } - val xact_init_data = (new FIFOIO) { new TransactionInitData } - val xact_abort = (new FIFOIO) { new TransactionAbort }.flip - val probe_req = (new FIFOIO) { new ProbeRequest }.flip - val probe_rep = (new FIFOIO) { new ProbeReply } - val probe_rep_data = (new FIFOIO) { new ProbeReplyData } - val xact_rep = (new FIFOIO) { new TransactionReply }.flip - val xact_finish = (new FIFOIO) { new TransactionFinish } - val incoherent = Bool(OUTPUT) -} - class XactTracker(id: Int, co: CoherencePolicy)(implicit conf: UncoreConfiguration) extends Component { val io = new Bundle { val alloc_req = (new FIFOIO) { new TrackerAllocReq }.flip From ffda0e41a99e3f344d08e43516ee446d0f28c4b0 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 18 Oct 2012 16:56:36 -0700 Subject: [PATCH 139/374] parameterize width of MemSerdes/MemDesser --- uncore/src/memserdes.scala | 54 ++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/uncore/src/memserdes.scala b/uncore/src/memserdes.scala index c44e2543..783fb3e1 100644 --- a/uncore/src/memserdes.scala +++ b/uncore/src/memserdes.scala @@ -6,17 +6,17 @@ import Constants._ import scala.math._ import uncore._ -class ioMemSerialized extends Bundle +class ioMemSerialized(w: Int) extends Bundle { - val req = (new FIFOIO) { Bits(width = MEM_BACKUP_WIDTH) } - val resp = (new PipeIO) { Bits(width = MEM_BACKUP_WIDTH) }.flip + val req = (new FIFOIO) { Bits(width = w) } + val resp = (new PipeIO) { Bits(width = w) }.flip } -class MemSerdes extends Component +class MemSerdes(w: Int) extends Component { val io = new Bundle { val wide = new ioMem().flip - val narrow = new ioMemSerialized + val narrow = new ioMemSerialized(w) } val abits = io.wide.req_cmd.bits.toBits.getWidth val dbits = io.wide.req_data.bits.toBits.getWidth @@ -27,14 +27,14 @@ class MemSerdes extends Component val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(5) { UFix() } val state = Reg(resetVal = s_idle) - val send_cnt = Reg(resetVal = UFix(0, log2Up((max(abits, dbits)+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) + val send_cnt = Reg(resetVal = UFix(0, log2Up((max(abits, dbits)+w-1)/w))) val data_send_cnt = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - val adone = io.narrow.req.ready && send_cnt === UFix((abits-1)/MEM_BACKUP_WIDTH) - val ddone = io.narrow.req.ready && send_cnt === UFix((dbits-1)/MEM_BACKUP_WIDTH) + val adone = io.narrow.req.ready && send_cnt === UFix((abits-1)/w) + val ddone = io.narrow.req.ready && send_cnt === UFix((dbits-1)/w) when (io.narrow.req.valid && io.narrow.req.ready) { send_cnt := send_cnt + UFix(1) - out_buf := out_buf >> UFix(MEM_BACKUP_WIDTH) + out_buf := out_buf >> UFix(w) } when (io.wide.req_cmd.valid && io.wide.req_cmd.ready) { out_buf := io.wide.req_cmd.bits.toBits @@ -68,19 +68,19 @@ class MemSerdes extends Component send_cnt := UFix(0) } - val recv_cnt = Reg(resetVal = UFix(0, log2Up((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) + val recv_cnt = Reg(resetVal = UFix(0, log2Up((rbits+w-1)/w))) val data_recv_cnt = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) val resp_val = Reg(resetVal = Bool(false)) resp_val := Bool(false) when (io.narrow.resp.valid) { recv_cnt := recv_cnt + UFix(1) - when (recv_cnt === UFix((rbits-1)/MEM_BACKUP_WIDTH)) { + when (recv_cnt === UFix((rbits-1)/w)) { recv_cnt := UFix(0) data_recv_cnt := data_recv_cnt + UFix(1) resp_val := Bool(true) } - in_buf := Cat(io.narrow.resp.bits, in_buf((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH*MEM_BACKUP_WIDTH-1,MEM_BACKUP_WIDTH)) + in_buf := Cat(io.narrow.resp.bits, in_buf((rbits+w-1)/w*w-1,w)) } io.wide.resp.valid := resp_val @@ -88,22 +88,24 @@ class MemSerdes extends Component io.wide.resp.bits.data := in_buf >> UFix(io.wide.resp.bits.tag.width) } -class MemDessert extends Component // test rig side +class MemDesserIO(w: Int) extends Bundle { + val narrow = new ioMemSerialized(w).flip + val wide = new ioMem +} + +class MemDesser(w: Int) extends Component // test rig side { - val io = new Bundle { - val narrow = new ioMemSerialized().flip - val wide = new ioMem - } + val io = new MemDesserIO(w) val abits = io.wide.req_cmd.bits.toBits.getWidth val dbits = io.wide.req_data.bits.toBits.getWidth val rbits = io.wide.resp.bits.getWidth require(dbits >= abits && rbits >= dbits) - val recv_cnt = Reg(resetVal = UFix(0, log2Up((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) + val recv_cnt = Reg(resetVal = UFix(0, log2Up((rbits+w-1)/w))) val data_recv_cnt = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - val adone = io.narrow.req.valid && recv_cnt === UFix((abits-1)/MEM_BACKUP_WIDTH) - val ddone = io.narrow.req.valid && recv_cnt === UFix((dbits-1)/MEM_BACKUP_WIDTH) - val rdone = io.narrow.resp.valid && recv_cnt === UFix((rbits-1)/MEM_BACKUP_WIDTH) + val adone = io.narrow.req.valid && recv_cnt === UFix((abits-1)/w) + val ddone = io.narrow.req.valid && recv_cnt === UFix((dbits-1)/w) + val rdone = io.narrow.resp.valid && recv_cnt === UFix((rbits-1)/w) val s_cmd_recv :: s_cmd :: s_data_recv :: s_data :: s_reply :: Nil = Enum(5) { UFix() } val state = Reg(resetVal = s_cmd_recv) @@ -111,7 +113,7 @@ class MemDessert extends Component // test rig side val in_buf = Reg() { Bits() } when (io.narrow.req.valid && io.narrow.req.ready || io.narrow.resp.valid) { recv_cnt := recv_cnt + UFix(1) - in_buf := Cat(io.narrow.req.bits, in_buf((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH*MEM_BACKUP_WIDTH-1,MEM_BACKUP_WIDTH)) + in_buf := Cat(io.narrow.req.bits, in_buf((rbits+w-1)/w*w-1,w)) } io.narrow.req.ready := state === s_cmd_recv || state === s_data_recv @@ -141,17 +143,17 @@ class MemDessert extends Component // test rig side data_recv_cnt := data_recv_cnt + UFix(1) } - val req_cmd = in_buf >> UFix(((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH - (abits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH)*MEM_BACKUP_WIDTH) + val req_cmd = in_buf >> UFix(((rbits+w-1)/w - (abits+w-1)/w)*w) io.wide.req_cmd.valid := state === s_cmd io.wide.req_cmd.bits := io.wide.req_cmd.bits.fromBits(req_cmd) io.wide.req_data.valid := state === s_data - io.wide.req_data.bits.data := in_buf >> UFix(((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH - (dbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH)*MEM_BACKUP_WIDTH) + io.wide.req_data.bits.data := in_buf >> UFix(((rbits+w-1)/w - (dbits+w-1)/w)*w) val dataq = (new Queue(REFILL_CYCLES)) { new MemResp } dataq.io.enq <> io.wide.resp - dataq.io.deq.ready := recv_cnt === UFix((rbits-1)/MEM_BACKUP_WIDTH) + dataq.io.deq.ready := recv_cnt === UFix((rbits-1)/w) io.narrow.resp.valid := dataq.io.deq.valid - io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UFix(MEM_BACKUP_WIDTH)) + io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UFix(w)) } From 2aecb0024fa12caa8a71aa1250bb25b432e0b8aa Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 18 Oct 2012 16:57:28 -0700 Subject: [PATCH 140/374] UncoreConfiguration now contains coherence policy --- uncore/src/consts.scala | 6 ------ uncore/src/uncore.scala | 16 ++++++++++------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index b7a70bac..757959fb 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -64,19 +64,13 @@ trait MemoryOpConstants { val M_XA_MAXU = Bits("b1111", 4); } -trait HTIFConstants { - val HTIF_WIDTH = 16 -} - trait MemoryInterfaceConstants extends - HTIFConstants with UncoreConstants with TileLinkSizeConstants { val MEM_TAG_BITS = max(TILE_XACT_ID_BITS, GLOBAL_XACT_ID_BITS) val MEM_DATA_BITS = 128 val REFILL_CYCLES = CACHE_DATA_SIZE_IN_BYTES*8/MEM_DATA_BITS - val MEM_BACKUP_WIDTH = HTIF_WIDTH } trait AddressConstants { diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index ebca0834..80c088f2 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -17,7 +17,8 @@ class TrackerDependency extends Bundle { val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } -class XactTracker(id: Int, co: CoherencePolicy)(implicit conf: UncoreConfiguration) extends Component { +class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component { + val co = conf.co val io = new Bundle { val alloc_req = (new FIFOIO) { new TrackerAllocReq }.flip val p_data = (new PipeIO) { new TrackerProbeData }.flip @@ -216,17 +217,19 @@ class XactTracker(id: Int, co: CoherencePolicy)(implicit conf: UncoreConfigurati } } -case class UncoreConfiguration(ntiles: Int, tile_id_bits: Int) +case class UncoreConfiguration(ntiles: Int, tile_id_bits: Int, co: CoherencePolicy) -abstract class CoherenceHub(co: CoherencePolicy)(implicit conf: UncoreConfiguration) extends Component { +abstract class CoherenceHub(implicit conf: UncoreConfiguration) extends Component { val io = new Bundle { val tiles = Vec(conf.ntiles) { new ioTileLink }.flip val mem = new ioMem } } -class CoherenceHubNull(co: ThreeStateIncoherence)(implicit conf: UncoreConfiguration) extends CoherenceHub(co)(conf) +class CoherenceHubNull(implicit conf: UncoreConfiguration) extends CoherenceHub { + val co = conf.co.asInstanceOf[ThreeStateIncoherence] + val x_init = io.tiles(0).xact_init val is_write = x_init.bits.x_type === co.xactInitWriteback x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp @@ -252,9 +255,10 @@ class CoherenceHubNull(co: ThreeStateIncoherence)(implicit conf: UncoreConfigura } -class CoherenceHubBroadcast(co: CoherencePolicy)(implicit conf: UncoreConfiguration) extends CoherenceHub(co)(conf) +class CoherenceHubBroadcast(implicit conf: UncoreConfiguration) extends CoherenceHub { - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_, co)) + val co = conf.co + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) val busy_arr = Vec(NGLOBAL_XACTS){ Bool() } val addr_arr = Vec(NGLOBAL_XACTS){ Bits(width=PADDR_BITS-OFFSET_BITS) } From 0cd0f8a9db89d34e057008d051b82a2e48ed6c23 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 23 Oct 2012 18:01:53 -0700 Subject: [PATCH 141/374] Initial version of migratory protocol --- uncore/src/coherence.scala | 174 ++++++++++++++++++++++++++++++++++++- 1 file changed, 172 insertions(+), 2 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index dcf384aa..7bbeaf48 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -9,7 +9,7 @@ object cpuCmdToRW { val load = (cmd === M_XRD) val amo = cmd(3).toBool val read = load || amo || (cmd === M_PFR) || (cmd === M_PFW) - val write = store || amo + val write = store || amo (read, write) } } @@ -556,7 +556,7 @@ class MESICoherence extends CoherencePolicyWithUncached { val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: Nil = Enum(8){ UFix() } val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() } val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() } - val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactReplyReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactInitReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = { @@ -699,3 +699,173 @@ class MESICoherence extends CoherencePolicyWithUncached { (x_type === xactInitWriteUncached) } } + +class MigratoryCoherence extends CoherencePolicyWithUncached { + + val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: tileSharedByTwo :: tileMigratoryClean :: tileMigratoryDirty :: Nil = Enum(7){ UFix() } + + val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: xactInitInvalidateOthers :: Nil = Enum(8){ UFix() } + val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: xactReplyReadMigratory :: Nil = Enum(9){ UFix() } + val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: probeReqInvalidateOthers :: Nil = Enum(4){ UFix() } + val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: probeRepDowngradeDataMigratory :: probeRepDowngradeAckHasCopy :: probeRepInvalidateDataMigratory :: probeRepInvalidateAckMigratory :: Nil = Enum(10){ UFix() } + val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactInitReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + + def uFixListContains(list: List[UFix], elem: UFix): Bool = list.map(elem === _).reduceLeft(_||_) + + def isHit (cmd: Bits, state: UFix): Bool = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, uFixListContains(List(tileExclusiveClean, tileExclusiveDirty, tileMigratoryClean, tileMigratoryDirty), state), (state != tileInvalid)) + } + def isValid (state: UFix): Bool = { + state != tileInvalid + } + + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { + val (read, write) = cpuCmdToRW(cmd) + (read && messageIsUncached(outstanding)) || + (write && (outstanding.x_type != xactInitReadExclusive && outstanding.x_type != xactInitInvalidateOthers)) + } + def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { + MuxLookup(cmd, (state === tileExclusiveDirty), Array( + M_INV -> uFixListContains(List(tileExclusiveDirty,tileMigratoryDirty),state), + M_CLN -> uFixListContains(List(tileExclusiveDirty,tileMigratoryDirty),state) + )) + } + def needsWriteback (state: UFix): Bool = { + needsTransactionOnCacheControl(M_INV, state) + } + + def newStateOnHit(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, MuxLookup(state, tileExclusiveDirty, Array( + tileExclusiveClean -> tileExclusiveDirty, + tileMigratoryClean -> tileMigratoryDirty)), state) + } + def newStateOnCacheControl(cmd: Bits) = { + MuxLookup(cmd, tileInvalid, Array( + M_INV -> tileInvalid, + M_CLN -> tileShared + )) + } + def newStateOnWriteback() = newStateOnCacheControl(M_INV) + def newStateOnFlush() = newStateOnCacheControl(M_INV) + def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { + MuxLookup(incoming.x_type, tileInvalid, Array( + xactReplyReadShared -> tileShared, + xactReplyReadExclusive -> MuxLookup(outstanding.x_type, tileExclusiveDirty, Array( + xactInitReadExclusive -> tileExclusiveDirty, + xactInitReadShared -> tileExclusiveClean)), + xactReplyReadExclusiveAck -> tileExclusiveDirty, + xactReplyReadUncached -> tileInvalid, + xactReplyWriteUncached -> tileInvalid, + xactReplyReadWordUncached -> tileInvalid, + xactReplyWriteWordUncached -> tileInvalid, + xactReplyAtomicUncached -> tileInvalid, + xactReplyReadMigratory -> MuxLookup(outstanding.x_type, tileMigratoryDirty, Array( + xactInitInvalidateOthers -> tileMigratoryDirty, + xactInitReadExclusive -> tileMigratoryDirty, + xactInitReadShared -> tileMigratoryClean)) + )) + } + def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { + MuxLookup(incoming.p_type, state, Array( + probeReqInvalidate -> tileInvalid, + probeReqInvalidateOthers -> tileInvalid, + probeReqCopy -> state, + probeReqDowngrade -> MuxLookup(state, tileShared, Array( + tileExclusiveClean -> tileSharedByTwo, + tileExclusiveDirty -> tileSharedByTwo, + tileSharedByTwo -> tileShared, + tileMigratoryClean -> tileSharedByTwo, + tileMigratoryDirty -> tileInvalid)) + )) + } + + def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id) + def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id) + def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) + def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) + def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) + + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write || cmd === M_PFW, Mux(state === tileInvalid, xactInitReadExclusive, xactInitInvalidateOthers), xactInitReadShared) + } + def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, Mux(state === tileInvalid, xactInitReadExclusive, xactInitInvalidateOthers), outstanding.x_type) + } + def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached + def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) + + def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { + Assert( incoming.p_type === probeReqInvalidateOthers && needsWriteback(state), "Bad probe request type, should be impossible.") + val reply = new ProbeReply() + val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( + probeReqInvalidate -> Mux(uFixListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), + probeRepInvalidateDataMigratory, probeRepInvalidateData), + probeReqDowngrade -> Mux(state === tileMigratoryDirty, probeRepDowngradeDataMigratory, probeRepDowngradeData), + probeReqCopy -> probeRepCopyData + )) + val without_data = MuxLookup(incoming.p_type, probeRepInvalidateAck, Array( + probeReqInvalidate -> Mux(tileExclusiveClean === state, probeRepInvalidateAckMigratory, probeRepInvalidateAck), + probeReqInvalidateOthers -> Mux(state === tileSharedByTwo, probeRepInvalidateAckMigratory, probeRepInvalidateAck), + probeReqDowngrade -> Mux(state != tileInvalid, probeRepDowngradeAckHasCopy, probeRepDowngradeAck), + probeReqCopy -> probeRepCopyAck + )) + reply.p_type := Mux(needsWriteback(state), with_data, without_data) + reply.global_xact_id := incoming.global_xact_id + reply + } + + def messageHasData (reply: ProbeReply): Bool = { + uFixListContains(List(probeRepInvalidateData, probeRepDowngradeData, probeRepCopyData, probeRepInvalidateDataMigratory, probeRepDowngradeDataMigratory), reply.p_type) + } + def messageHasData (init: TransactionInit): Bool = uFixListContains(hasDataTypeList, init.x_type) + def messageHasData (reply: TransactionReply): Bool = { + uFixListContains(List(xactReplyReadShared, xactReplyReadExclusive, xactReplyReadUncached, xactReplyReadMigratory, xactReplyReadWordUncached, xactReplyAtomicUncached), reply.x_type) + } + def messageUpdatesDataArray (reply: TransactionReply): Bool = { + uFixListContains(List(xactReplyReadShared, xactReplyReadExclusive, xactReplyReadMigratory), reply.x_type) + } + def messageIsUncached(init: TransactionInit): Bool = uFixListContains(uncachedTypeList, init.x_type) + + def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) + + def getTransactionReplyType(x_type: UFix, count: UFix): Bits = { + MuxLookup(x_type, xactReplyReadUncached, Array( + xactInitReadShared -> Mux(count > UFix(0), xactReplyReadShared, xactReplyReadExclusive), //TODO: what is count? Depend on probeRep.p_type??? + xactInitReadExclusive -> xactReplyReadExclusive, + xactInitReadUncached -> xactReplyReadUncached, + xactInitWriteUncached -> xactReplyWriteUncached, + xactInitReadWordUncached -> xactReplyReadWordUncached, + xactInitWriteWordUncached -> xactReplyWriteWordUncached, + xactInitAtomicUncached -> xactReplyAtomicUncached, + xactInitInvalidateOthers -> xactReplyReadExclusiveAck //TODO: add this to MESI? + )) + } + + def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = { + MuxLookup(x_type, probeReqCopy, Array( + xactInitReadShared -> probeReqDowngrade, + xactInitReadExclusive -> probeReqInvalidate, + xactInitReadUncached -> probeReqCopy, + xactInitWriteUncached -> probeReqInvalidate, + xactInitReadWordUncached -> probeReqCopy, + xactInitWriteWordUncached -> probeReqInvalidate, + xactInitAtomicUncached -> probeReqInvalidate, + xactInitInvalidateOthers -> probeReqInvalidateOthers + )) + } + + def needsMemRead(x_type: UFix, global_state: UFix): Bool = { + (x_type != xactInitWriteUncached && x_type != xactInitInvalidateOthers) + } + def needsMemWrite(x_type: UFix, global_state: UFix): Bool = { + (x_type === xactInitWriteUncached || x_type === xactInitWriteWordUncached || x_type === xactInitAtomicUncached) + } + def needsAckReply(x_type: UFix, global_state: UFix): Bool = { + (x_type === xactInitWriteUncached || x_type === xactInitWriteWordUncached ||x_type === xactInitInvalidateOthers) + } +} From 3e6dc35809509ddffac67a7d4af1d2c0f2294eff Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 16 Nov 2012 02:37:56 -0800 Subject: [PATCH 142/374] issue self-probes for uncached read transactions this facilitates I$ coherence. but it seems like a hack and perhaps the mechanism should be rethought. --- uncore/src/coherence.scala | 15 +++++++-------- uncore/src/consts.scala | 5 ++++- uncore/src/uncore.scala | 34 ++++++++++++++++++++-------------- 3 files changed, 31 insertions(+), 23 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index 7bbeaf48..d7cf1ded 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -4,14 +4,7 @@ import Chisel._ import Constants._ object cpuCmdToRW { - def apply(cmd: Bits): (Bool, Bool) = { - val store = (cmd === M_XWR) - val load = (cmd === M_XRD) - val amo = cmd(3).toBool - val read = load || amo || (cmd === M_PFR) || (cmd === M_PFW) - val write = store || amo - (read, write) - } + def apply(cmd: Bits): (Bool, Bool) = (isRead(cmd), isWrite(cmd)) } abstract class CoherencePolicy { @@ -56,6 +49,7 @@ trait UncachedTransactions { def getUncachedReadWordTransactionInit(addr: UFix, id: UFix): TransactionInit def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits): TransactionInit def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix): TransactionInit + def isUncachedReadTransaction(xinit: TransactionInit): Bool } abstract class CoherencePolicyWithUncached extends CoherencePolicy with UncachedTransactions @@ -182,6 +176,7 @@ class MICoherence extends CoherencePolicyWithUncached { def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) + def isUncachedReadTransaction(xinit: TransactionInit) = xinit.x_type === xactInitReadUncached def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = xactInitReadExclusive def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = xactInitReadExclusive @@ -317,6 +312,7 @@ class MEICoherence extends CoherencePolicyWithUncached { def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) + def isUncachedReadTransaction(xinit: TransactionInit) = xinit.x_type === xactInitReadUncached def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -470,6 +466,7 @@ class MSICoherence extends CoherencePolicyWithUncached { def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) + def isUncachedReadTransaction(xinit: TransactionInit) = xinit.x_type === xactInitReadUncached def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -620,6 +617,7 @@ class MESICoherence extends CoherencePolicyWithUncached { def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) + def isUncachedReadTransaction(xinit: TransactionInit) = xinit.x_type === xactInitReadUncached def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -787,6 +785,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) + def isUncachedReadTransaction(xinit: TransactionInit) = xinit.x_type === xactInitReadUncached def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index 757959fb..57390387 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -50,7 +50,6 @@ trait MemoryOpConstants { val M_XWR = Bits("b0001", 4); // int store val M_PFR = Bits("b0010", 4); // prefetch with intent to read val M_PFW = Bits("b0011", 4); // prefetch with intent to write - val M_FLA = Bits("b0100", 4); // write back and invlaidate all lines val M_FENCE = Bits("b0101", 4); // memory fence val M_INV = Bits("b0110", 4); // write back and invalidate line val M_CLN = Bits("b0111", 4); // write back line @@ -62,6 +61,10 @@ trait MemoryOpConstants { val M_XA_MAX = Bits("b1101", 4); val M_XA_MINU = Bits("b1110", 4); val M_XA_MAXU = Bits("b1111", 4); + + def isAMO(cmd: Bits) = cmd(3) + def isRead(cmd: Bits) = cmd === M_XRD || cmd === M_PFR || cmd === M_PFW || isAMO(cmd) + def isWrite(cmd: Bits) = cmd === M_XWR || isAMO(cmd) } trait MemoryInterfaceConstants extends diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 80c088f2..e2dcfc30 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -89,10 +89,8 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) - val addr_ = Reg{ UFix() } - val x_type_ = Reg{ Bits() } + val xact = Reg{ new TransactionInit } val init_tile_id_ = Reg{ Bits() } - val tile_xact_id_ = Reg{ Bits() } val p_rep_count = if (conf.ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ntiles))) val p_req_flags = Reg(resetVal = Bits(0, width = conf.ntiles)) val p_rep_tile_id_ = Reg{ Bits() } @@ -105,27 +103,37 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) val p_req_initial_flags = Bits(width = conf.ntiles) - p_req_initial_flags := (if (conf.ntiles == 1) Bits(0) else ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id(log2Up(conf.ntiles)-1,0)))) //TODO: Broadcast only + p_req_initial_flags := Bits(0) + if (conf.ntiles > 1) { + // issue self-probes for uncached read xacts to facilitate I$ coherence + // TODO: this is hackish; figure out how to do it more systematically + val probe_self = co match { + case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.alloc_req.bits.xact_init) + case _ => Bool(false) + } + val myflag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.tile_id(log2Up(conf.ntiles)-1,0))) + p_req_initial_flags := ~(io.tile_incoherent | myflag) + } io.busy := state != s_idle - io.addr := addr_ + io.addr := xact.addr io.init_tile_id := init_tile_id_ io.p_rep_tile_id := p_rep_tile_id_ - io.tile_xact_id := tile_xact_id_ + io.tile_xact_id := xact.tile_xact_id io.sharer_count := UFix(conf.ntiles) // TODO: Broadcast only - io.x_type := x_type_ + io.x_type := xact.x_type io.mem_req_cmd.valid := Bool(false) io.mem_req_cmd.bits.rw := Bool(false) - io.mem_req_cmd.bits.addr := addr_ + io.mem_req_cmd.bits.addr := xact.addr io.mem_req_cmd.bits.tag := UFix(id) io.mem_req_data.valid := Bool(false) io.mem_req_data.bits.data := UFix(0) io.mem_req_lock := Bool(false) io.probe_req.valid := Bool(false) - io.probe_req.bits.p_type := co.getProbeRequestType(x_type_, UFix(0)) + io.probe_req.bits.p_type := co.getProbeRequestType(xact.x_type, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) - io.probe_req.bits.addr := addr_ + io.probe_req.bits.addr := xact.addr io.push_p_req := Bits(0, width = conf.ntiles) io.pop_p_rep := Bits(0, width = conf.ntiles) io.pop_p_rep_data := Bits(0, width = conf.ntiles) @@ -138,10 +146,8 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component switch (state) { is(s_idle) { when( io.alloc_req.valid && io.can_alloc ) { - addr_ := io.alloc_req.bits.xact_init.addr - x_type_ := io.alloc_req.bits.xact_init.x_type + xact := io.alloc_req.bits.xact_init init_tile_id_ := io.alloc_req.bits.tile_id - tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) p_req_flags := p_req_initial_flags @@ -202,7 +208,7 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { - state := Mux(co.needsAckReply(x_type_, UFix(0)), s_ack, s_busy) + state := Mux(co.needsAckReply(xact.x_type, UFix(0)), s_ack, s_busy) } } is(s_ack) { From 6bd4f93f8c27044e6dd6244258b3c4082deda354 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Sun, 18 Nov 2012 03:13:17 -0800 Subject: [PATCH 143/374] pull out prefetch commands from isRead --- uncore/src/consts.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index 57390387..29be0acc 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -63,7 +63,8 @@ trait MemoryOpConstants { val M_XA_MAXU = Bits("b1111", 4); def isAMO(cmd: Bits) = cmd(3) - def isRead(cmd: Bits) = cmd === M_XRD || cmd === M_PFR || cmd === M_PFW || isAMO(cmd) + def isPrefetch(cmd: Bits) = cmd === M_PFR || cmd === M_PFW + def isRead(cmd: Bits) = cmd === M_XRD || isAMO(cmd) def isWrite(cmd: Bits) = cmd === M_XWR || isAMO(cmd) } From 56f9b9721df5283e0fb3f854cf6eb42a52f5bbca Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 20 Nov 2012 05:38:49 -0800 Subject: [PATCH 144/374] treat prefetches as read requests --- uncore/src/coherence.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index d7cf1ded..e8aad9d0 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -4,7 +4,7 @@ import Chisel._ import Constants._ object cpuCmdToRW { - def apply(cmd: Bits): (Bool, Bool) = (isRead(cmd), isWrite(cmd)) + def apply(cmd: Bits): (Bool, Bool) = (isRead(cmd) || isPrefetch(cmd), isWrite(cmd)) } abstract class CoherencePolicy { From 8103676b37b40f0a32f46527ad771f3dc0edf54b Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 26 Nov 2012 20:54:56 -0800 Subject: [PATCH 145/374] reduce physical address space to 4GB --- uncore/src/consts.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index 29be0acc..0b885352 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -78,7 +78,7 @@ trait MemoryInterfaceConstants extends } trait AddressConstants { - val PADDR_BITS = 40; + val PADDR_BITS = 32 val VADDR_BITS = 43; val PGIDX_BITS = 13; val PPN_BITS = PADDR_BITS-PGIDX_BITS; From 50e9d952e874ae5415d87014966a350347ec23e1 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 4 Dec 2012 06:57:53 -0800 Subject: [PATCH 146/374] don't initiate llc refill until writeback drains --- uncore/src/llc.scala | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index cb3fbb6a..2d3eea16 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -109,6 +109,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val requested = Bool() val old_dirty = Bool() val old_tag = UFix(width = PADDR_BITS - OFFSET_BITS - log2Up(sets)) + val wb_busy = Bool() override def clone = new MSHR().asInstanceOf[this.type] } @@ -124,12 +125,13 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component mshr(freeId).way := io.repl_way mshr(freeId).old_dirty := io.repl_dirty mshr(freeId).old_tag := io.repl_tag + mshr(freeId).wb_busy := Bool(false) mshr(freeId).requested := Bool(false) mshr(freeId).refillCount := UFix(0) mshr(freeId).refilled := Bool(false) } - val requests = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && !mshr(i).old_dirty && !mshr(i).requested):_*) + val requests = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && !mshr(i).old_dirty && !mshr(i).wb_busy && !mshr(i).requested):_*) val request = requests.orR val requestId = PriorityEncoder(requests) when (io.mem.req_cmd.valid && io.mem.req_cmd.ready) { mshr(requestId).requested := Bool(true) } @@ -149,7 +151,11 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val writebacks = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).old_dirty):_*) val writeback = writebacks.orR val writebackId = PriorityEncoder(writebacks) - when (writeback && io.data.ready && !replay) { mshr(writebackId).old_dirty := Bool(false) } + when (writeback && io.data.ready && !replay) { + mshr(writebackId).old_dirty := Bool(false) + mshr(writebackId).wb_busy := Bool(true) + } + mshr.foreach(m => when (m.wb_busy && io.data.ready) { m.wb_busy := Bool(false) }) val conflicts = Cat(Bits(0), (0 until outstanding).map(i => valid(i) && io.cpu.bits.addr(log2Up(sets)-1, 0) === mshr(i).addr(log2Up(sets)-1, 0)):_*) io.cpu.ready := !conflicts.orR && !validBits.andR From aae7a677812e20227d8f6f87b56c40765dbd91f5 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 6 Dec 2012 02:07:03 -0800 Subject: [PATCH 147/374] fix llc refill/writeback bugs --- uncore/src/llc.scala | 55 ++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index 2d3eea16..7ccdedb8 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -132,7 +132,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component } val requests = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && !mshr(i).old_dirty && !mshr(i).wb_busy && !mshr(i).requested):_*) - val request = requests.orR + val request = requests.orR && io.data.ready // allow in-flight hits to drain val requestId = PriorityEncoder(requests) when (io.mem.req_cmd.valid && io.mem.req_cmd.ready) { mshr(requestId).requested := Bool(true) } @@ -365,19 +365,21 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val initialize = !initCount(log2Up(sets)) when (initialize) { initCount := initCount + UFix(1) } - val stall_s1 = Bool() - val replay_s1 = Reg(resetVal = Bool(false)) - val s1_valid = Reg(io.cpu.req_cmd.fire() || replay_s1, resetVal = Bool(false)) - replay_s1 := s1_valid && stall_s1 + val replay_s2 = Reg(resetVal = Bool(false)) + val s2_valid = Reg(resetVal = Bool(false)) + val s2 = Reg() { new MemReqCmd } + val s3_rdy = Bool() + val replay_s2_rdy = Bool() + + val s1_valid = Reg(io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy, resetVal = Bool(false)) val s1 = Reg() { new MemReqCmd } when (io.cpu.req_cmd.fire()) { s1 := io.cpu.req_cmd.bits } + when (replay_s2 && replay_s2_rdy) { s1 := s2 } - val stall_s2 = Bool() - val s2_valid = Reg(resetVal = Bool(false)) - s2_valid := s1_valid && !replay_s1 && !stall_s1 || stall_s2 - val s2 = Reg() { new MemReqCmd } + s2_valid := s1_valid + replay_s2 := s2_valid && !s3_rdy || replay_s2 && !replay_s2_rdy val s2_tags = Vec(ways) { Reg() { Bits(width = metaWidth) } } - when (s1_valid && !stall_s1 && !replay_s1) { + when (s1_valid) { s2 := s1 for (i <- 0 until ways) s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) @@ -389,26 +391,24 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val repl_way = LFSR16(s2_valid)(log2Up(ways)-1, 0) val repl_tag = s2_tags(repl_way).toUFix val setDirty = s2_valid && s2.rw && s2_hit && !s2_hit_dirty - stall_s1 := initialize || stall_s2 - val tag_we = setDirty || mshr.io.tag.valid - val tag_waddr = Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(log2Up(sets)-1,0) - val tag_wdata = Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)) - val tag_wway = Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way) - tags.io.in.valid := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || tag_we - tags.io.in.bits.addr := Mux(initialize, initCount, Mux(tag_we, tag_waddr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)(log2Up(sets)-1,0))) - tags.io.in.bits.rw := initialize || tag_we - tags.io.in.bits.wdata := Mux(initialize, UFix(0), Fill(ways, tag_wdata)) - tags.io.in.bits.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(tag_wway))) - when (tag_we && Mux(stall_s2, s2.addr, s1.addr)(log2Up(sets)-1,0) === tag_waddr) { s2_tags(tag_wway) := tag_wdata } + val tag_we = initialize || setDirty || mshr.io.tag.fire() + val tag_waddr = Mux(initialize, initCount, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)) + val tag_wdata = Cat(setDirty, !initialize, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)) + val tag_wmask = Mux(initialize, Fix(-1, ways), UFixToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way))) + tags.io.in.valid := io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy || tag_we + tags.io.in.bits.addr := Mux(tag_we, tag_waddr, Mux(replay_s2, s2.addr, io.cpu.req_cmd.bits.addr)(log2Up(sets)-1,0)) + tags.io.in.bits.rw := tag_we + tags.io.in.bits.wdata := Fill(ways, tag_wdata) + tags.io.in.bits.wmask := FillInterleaved(metaWidth, tag_wmask) - mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw && dataArb.io.in(1).ready && writeback.io.req(0).ready // stall_s2 + mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw mshr.io.cpu.bits := s2 mshr.io.repl_way := repl_way mshr.io.repl_dirty := repl_tag(tagWidth+1, tagWidth).andR mshr.io.repl_tag := repl_tag mshr.io.mem.resp := io.mem.resp - mshr.io.tag.ready := !setDirty + mshr.io.tag.ready := !s1_valid && !s2_valid data.io.req <> dataArb.io.out data.io.mem_resp := io.mem.resp @@ -419,7 +419,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da writeback.io.req(0) <> data.io.writeback writeback.io.data(0) <> data.io.writeback_data - writeback.io.req(1).valid := s2_valid && !s2_hit && s2.rw && dataArb.io.in(1).ready && mshr.io.cpu.ready // stall_s2 + writeback.io.req(1).valid := s2_valid && !s2_hit && s2.rw && mshr.io.cpu.ready writeback.io.req(1).bits := s2.addr writeback.io.data(1).valid := io.cpu.req_data.valid writeback.io.data(1).bits := io.cpu.req_data.bits @@ -428,15 +428,16 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da memCmdArb.io.in(1) <> writeback.io.mem.req_cmd dataArb.io.in(0) <> mshr.io.data - dataArb.io.in(1).valid := s2_valid && s2_hit && writeback.io.req(0).ready && mshr.io.cpu.ready // stall_s2 + dataArb.io.in(1).valid := s2_valid && s2_hit && mshr.io.cpu.ready dataArb.io.in(1).bits := s2 dataArb.io.in(1).bits.way := s2_hit_way dataArb.io.in(1).bits.isWriteback := Bool(false) - stall_s2 := s2_valid && !(dataArb.io.in(1).ready && writeback.io.req(0).ready && mshr.io.cpu.ready) + s3_rdy := mshr.io.cpu.ready && Mux(s2_hit, dataArb.io.in(1).ready, !s2.rw || writeback.io.req(1).ready) + replay_s2_rdy := s3_rdy && !tag_we io.cpu.resp <> data.io.resp - io.cpu.req_cmd.ready := !stall_s1 && !replay_s1 && !tag_we + io.cpu.req_cmd.ready := !s1_valid && !s2_valid && !replay_s2 && !tag_we io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready io.mem.req_cmd <> memCmdArb.io.out io.mem.req_data <> writeback.io.mem.req_data From 12caa55dc7e7aa99c4b2449d01fa773d07b8d863 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 6 Dec 2012 18:51:30 -0800 Subject: [PATCH 148/374] wip: new network classes --- uncore/src/tilelink.scala | 66 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index bddbc0d3..00d40e54 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -3,6 +3,72 @@ package uncore import Chisel._ import Constants._ +case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) + +class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { + val src = UFix(width = conf.idBits) + val dst = UFix(width = conf.idBits) +} + +class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetworkIO()(data)(conf) { + val temp = UFix(width = conf.idBits) +} + +abstract class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends FIFOIO()(data) { + val header = (new PhysicalHeader).asInput +} + + +abstract class PhysicalNetwork(implicit conf: PhysicalNetworkConfiguration) extends Component + +class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { + val io = new Bundle { + val in = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } } + val out = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } }.flip + } + + for(i <- 0 until conf.nEndpoints) { + val rrarb = new RRArbiter(conf.nEndpoints)(data) + (rrarb.io.in, io.in).zipped.map( (arb, io) => { + arb.valid := io.valid && (io.header.dst === UFix(i)) + arb.bits := io.bits + io.ready := arb.ready + }) + io.out(i) <> rrarb.io.out + } +} + +case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int) + +abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[Component])(implicit conf: LogicalNetworkConfiguration) extends Component { + val io: Vec[TileLinkType] + val physicalNetworks: Seq[PhysicalNetwork] + require(endpoints.length == conf.nEndpoints) +} + +class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val src = UFix(width = conf.idBits) + val dst = UFix(width = conf.idBits) +} + +abstract class LogicalNetworkIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends FIFOIO()(data) { + val header = (new LogicalHeader).asInput +} + +class TileIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(m,conf) +class HubIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(m,conf) + +class TileLink(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val xact_init = (new TileIO) { new TransactionInit } + val xact_init_data = (new TileIO) { new TransactionInitData } + val xact_abort = (new HubIO) { new TransactionAbort } + val probe_req = (new HubIO) { new ProbeRequest } + val probe_rep = (new TileIO) { new ProbeReply } + val probe_rep_data = (new TileIO) { new ProbeReplyData } + val xact_rep = (new HubIO) { new TransactionReply } + val xact_finish = (new TileIO) { new TransactionFinish } + val incoherent = Bool(OUTPUT) +} class PhysicalAddress extends Bundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) From 8d69f9b41cebb267d5e8cab7756fb2a6e616e10f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 12 Dec 2012 00:05:28 -0800 Subject: [PATCH 149/374] Initial version of phys/log network compiles --- uncore/src/memserdes.scala | 1 - uncore/src/package.scala | 74 ++++++++++++++++++++++++++++++++------ uncore/src/temp.scala | 13 +++++++ uncore/src/tilelink.scala | 67 ---------------------------------- 4 files changed, 77 insertions(+), 78 deletions(-) create mode 100644 uncore/src/temp.scala diff --git a/uncore/src/memserdes.scala b/uncore/src/memserdes.scala index 783fb3e1..88a6f1b3 100644 --- a/uncore/src/memserdes.scala +++ b/uncore/src/memserdes.scala @@ -4,7 +4,6 @@ import Chisel._ import Node._ import Constants._ import scala.math._ -import uncore._ class ioMemSerialized(w: Int) extends Bundle { diff --git a/uncore/src/package.scala b/uncore/src/package.scala index e3a8f554..a3af64e1 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -1,13 +1,67 @@ -package uncore -import uncore.constants._ +package object uncore { +import Chisel._ -//TODO: When compiler bug SI-5604 is fixed in 2.10, change object Constants to -// package object uncore and remove import Constants._'s from other files -object Constants extends - MemoryOpConstants with - MemoryInterfaceConstants with - CacheConstants with - AddressConstants -{ +case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) + +class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { + val src = UFix(width = conf.idBits) + val dst = UFix(width = conf.idBits) } +abstract class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends FIFOIO()(data) { + val header = (new PhysicalHeader).asOutput +} + +class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetworkIO()(data)(conf) + +abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Component + +class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { + val io = new Bundle { + val in = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } }.flip + val out = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } } + } + + for(i <- 0 until conf.nEndpoints) { + val rrarb = new RRArbiter(conf.nEndpoints)(data) + (rrarb.io.in, io.in).zipped.map( (arb, io) => { + arb.valid := io.valid && (io.header.dst === UFix(i)) + arb.bits := io.bits + io.ready := arb.ready + }) + io.out(i) <> rrarb.io.out + } +} + +case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nHubs: Int, nTiles: Int) + +abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[Component])(implicit conf: LogicalNetworkConfiguration) extends Component { + val io: Vec[TileLinkType] + val physicalNetworks: Seq[PhysicalNetwork] + require(endpoints.length == conf.nEndpoints) +} + +class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val src = UFix(width = conf.idBits) + val dst = UFix(width = conf.idBits) +} + +abstract class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends FIFOIO()(data) { + val header = (new LogicalHeader).asOutput +} + +class TileIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(conf) +class HubIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(conf){flip()} + +class TileLink(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val xact_init = (new TileIO) { new TransactionInit } + val xact_init_data = (new TileIO) { new TransactionInitData } + val xact_abort = (new HubIO) { new TransactionAbort } + val probe_req = (new HubIO) { new ProbeRequest } + val probe_rep = (new TileIO) { new ProbeReply } + val probe_rep_data = (new TileIO) { new ProbeReplyData } + val xact_rep = (new HubIO) { new TransactionReply } + val xact_finish = (new TileIO) { new TransactionFinish } + val incoherent = Bool(OUTPUT) +} +} diff --git a/uncore/src/temp.scala b/uncore/src/temp.scala new file mode 100644 index 00000000..b2d3fdca --- /dev/null +++ b/uncore/src/temp.scala @@ -0,0 +1,13 @@ +package uncore +import _root_.uncore.constants._ + +//TODO: When compiler bug SI-5604 is fixed in 2.10, remove object Constants and +// mixin Constants traits to package object uncore in package.scala and +// remove import Constants._'s from other .scala files +object Constants extends + MemoryOpConstants with + MemoryInterfaceConstants with + CacheConstants with + AddressConstants +{ +} diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index 00d40e54..a9cbc482 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -3,73 +3,6 @@ package uncore import Chisel._ import Constants._ -case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) - -class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { - val src = UFix(width = conf.idBits) - val dst = UFix(width = conf.idBits) -} - -class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetworkIO()(data)(conf) { - val temp = UFix(width = conf.idBits) -} - -abstract class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends FIFOIO()(data) { - val header = (new PhysicalHeader).asInput -} - - -abstract class PhysicalNetwork(implicit conf: PhysicalNetworkConfiguration) extends Component - -class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { - val io = new Bundle { - val in = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } } - val out = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } }.flip - } - - for(i <- 0 until conf.nEndpoints) { - val rrarb = new RRArbiter(conf.nEndpoints)(data) - (rrarb.io.in, io.in).zipped.map( (arb, io) => { - arb.valid := io.valid && (io.header.dst === UFix(i)) - arb.bits := io.bits - io.ready := arb.ready - }) - io.out(i) <> rrarb.io.out - } -} - -case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int) - -abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[Component])(implicit conf: LogicalNetworkConfiguration) extends Component { - val io: Vec[TileLinkType] - val physicalNetworks: Seq[PhysicalNetwork] - require(endpoints.length == conf.nEndpoints) -} - -class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val src = UFix(width = conf.idBits) - val dst = UFix(width = conf.idBits) -} - -abstract class LogicalNetworkIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends FIFOIO()(data) { - val header = (new LogicalHeader).asInput -} - -class TileIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(m,conf) -class HubIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(m,conf) - -class TileLink(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val xact_init = (new TileIO) { new TransactionInit } - val xact_init_data = (new TileIO) { new TransactionInitData } - val xact_abort = (new HubIO) { new TransactionAbort } - val probe_req = (new HubIO) { new ProbeRequest } - val probe_rep = (new TileIO) { new ProbeReply } - val probe_rep_data = (new TileIO) { new ProbeReplyData } - val xact_rep = (new HubIO) { new TransactionReply } - val xact_finish = (new TileIO) { new TransactionFinish } - val incoherent = Bool(OUTPUT) -} - class PhysicalAddress extends Bundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) } From f359518e52375496f6cde523046310b215bc5955 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 6 Dec 2012 18:51:30 -0800 Subject: [PATCH 150/374] wip: new network classes --- uncore/src/tilelink.scala | 66 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index bddbc0d3..00d40e54 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -3,6 +3,72 @@ package uncore import Chisel._ import Constants._ +case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) + +class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { + val src = UFix(width = conf.idBits) + val dst = UFix(width = conf.idBits) +} + +class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetworkIO()(data)(conf) { + val temp = UFix(width = conf.idBits) +} + +abstract class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends FIFOIO()(data) { + val header = (new PhysicalHeader).asInput +} + + +abstract class PhysicalNetwork(implicit conf: PhysicalNetworkConfiguration) extends Component + +class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { + val io = new Bundle { + val in = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } } + val out = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } }.flip + } + + for(i <- 0 until conf.nEndpoints) { + val rrarb = new RRArbiter(conf.nEndpoints)(data) + (rrarb.io.in, io.in).zipped.map( (arb, io) => { + arb.valid := io.valid && (io.header.dst === UFix(i)) + arb.bits := io.bits + io.ready := arb.ready + }) + io.out(i) <> rrarb.io.out + } +} + +case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int) + +abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[Component])(implicit conf: LogicalNetworkConfiguration) extends Component { + val io: Vec[TileLinkType] + val physicalNetworks: Seq[PhysicalNetwork] + require(endpoints.length == conf.nEndpoints) +} + +class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val src = UFix(width = conf.idBits) + val dst = UFix(width = conf.idBits) +} + +abstract class LogicalNetworkIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends FIFOIO()(data) { + val header = (new LogicalHeader).asInput +} + +class TileIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(m,conf) +class HubIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(m,conf) + +class TileLink(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val xact_init = (new TileIO) { new TransactionInit } + val xact_init_data = (new TileIO) { new TransactionInitData } + val xact_abort = (new HubIO) { new TransactionAbort } + val probe_req = (new HubIO) { new ProbeRequest } + val probe_rep = (new TileIO) { new ProbeReply } + val probe_rep_data = (new TileIO) { new ProbeReplyData } + val xact_rep = (new HubIO) { new TransactionReply } + val xact_finish = (new TileIO) { new TransactionFinish } + val incoherent = Bool(OUTPUT) +} class PhysicalAddress extends Bundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) From 6d61baa6cd8d977bae9a93e862bb3972017eb6f2 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 12 Dec 2012 00:05:28 -0800 Subject: [PATCH 151/374] Initial version of phys/log network compiles --- uncore/src/memserdes.scala | 1 - uncore/src/package.scala | 74 ++++++++++++++++++++++++++++++++------ uncore/src/temp.scala | 13 +++++++ uncore/src/tilelink.scala | 67 ---------------------------------- 4 files changed, 77 insertions(+), 78 deletions(-) create mode 100644 uncore/src/temp.scala diff --git a/uncore/src/memserdes.scala b/uncore/src/memserdes.scala index 783fb3e1..88a6f1b3 100644 --- a/uncore/src/memserdes.scala +++ b/uncore/src/memserdes.scala @@ -4,7 +4,6 @@ import Chisel._ import Node._ import Constants._ import scala.math._ -import uncore._ class ioMemSerialized(w: Int) extends Bundle { diff --git a/uncore/src/package.scala b/uncore/src/package.scala index e3a8f554..a3af64e1 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -1,13 +1,67 @@ -package uncore -import uncore.constants._ +package object uncore { +import Chisel._ -//TODO: When compiler bug SI-5604 is fixed in 2.10, change object Constants to -// package object uncore and remove import Constants._'s from other files -object Constants extends - MemoryOpConstants with - MemoryInterfaceConstants with - CacheConstants with - AddressConstants -{ +case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) + +class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { + val src = UFix(width = conf.idBits) + val dst = UFix(width = conf.idBits) } +abstract class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends FIFOIO()(data) { + val header = (new PhysicalHeader).asOutput +} + +class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetworkIO()(data)(conf) + +abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Component + +class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { + val io = new Bundle { + val in = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } }.flip + val out = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } } + } + + for(i <- 0 until conf.nEndpoints) { + val rrarb = new RRArbiter(conf.nEndpoints)(data) + (rrarb.io.in, io.in).zipped.map( (arb, io) => { + arb.valid := io.valid && (io.header.dst === UFix(i)) + arb.bits := io.bits + io.ready := arb.ready + }) + io.out(i) <> rrarb.io.out + } +} + +case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nHubs: Int, nTiles: Int) + +abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[Component])(implicit conf: LogicalNetworkConfiguration) extends Component { + val io: Vec[TileLinkType] + val physicalNetworks: Seq[PhysicalNetwork] + require(endpoints.length == conf.nEndpoints) +} + +class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val src = UFix(width = conf.idBits) + val dst = UFix(width = conf.idBits) +} + +abstract class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends FIFOIO()(data) { + val header = (new LogicalHeader).asOutput +} + +class TileIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(conf) +class HubIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(conf){flip()} + +class TileLink(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val xact_init = (new TileIO) { new TransactionInit } + val xact_init_data = (new TileIO) { new TransactionInitData } + val xact_abort = (new HubIO) { new TransactionAbort } + val probe_req = (new HubIO) { new ProbeRequest } + val probe_rep = (new TileIO) { new ProbeReply } + val probe_rep_data = (new TileIO) { new ProbeReplyData } + val xact_rep = (new HubIO) { new TransactionReply } + val xact_finish = (new TileIO) { new TransactionFinish } + val incoherent = Bool(OUTPUT) +} +} diff --git a/uncore/src/temp.scala b/uncore/src/temp.scala new file mode 100644 index 00000000..b2d3fdca --- /dev/null +++ b/uncore/src/temp.scala @@ -0,0 +1,13 @@ +package uncore +import _root_.uncore.constants._ + +//TODO: When compiler bug SI-5604 is fixed in 2.10, remove object Constants and +// mixin Constants traits to package object uncore in package.scala and +// remove import Constants._'s from other .scala files +object Constants extends + MemoryOpConstants with + MemoryInterfaceConstants with + CacheConstants with + AddressConstants +{ +} diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index 00d40e54..a9cbc482 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -3,73 +3,6 @@ package uncore import Chisel._ import Constants._ -case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) - -class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { - val src = UFix(width = conf.idBits) - val dst = UFix(width = conf.idBits) -} - -class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetworkIO()(data)(conf) { - val temp = UFix(width = conf.idBits) -} - -abstract class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends FIFOIO()(data) { - val header = (new PhysicalHeader).asInput -} - - -abstract class PhysicalNetwork(implicit conf: PhysicalNetworkConfiguration) extends Component - -class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { - val io = new Bundle { - val in = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } } - val out = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } }.flip - } - - for(i <- 0 until conf.nEndpoints) { - val rrarb = new RRArbiter(conf.nEndpoints)(data) - (rrarb.io.in, io.in).zipped.map( (arb, io) => { - arb.valid := io.valid && (io.header.dst === UFix(i)) - arb.bits := io.bits - io.ready := arb.ready - }) - io.out(i) <> rrarb.io.out - } -} - -case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int) - -abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[Component])(implicit conf: LogicalNetworkConfiguration) extends Component { - val io: Vec[TileLinkType] - val physicalNetworks: Seq[PhysicalNetwork] - require(endpoints.length == conf.nEndpoints) -} - -class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val src = UFix(width = conf.idBits) - val dst = UFix(width = conf.idBits) -} - -abstract class LogicalNetworkIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends FIFOIO()(data) { - val header = (new LogicalHeader).asInput -} - -class TileIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(m,conf) -class HubIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(m,conf) - -class TileLink(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val xact_init = (new TileIO) { new TransactionInit } - val xact_init_data = (new TileIO) { new TransactionInitData } - val xact_abort = (new HubIO) { new TransactionAbort } - val probe_req = (new HubIO) { new ProbeRequest } - val probe_rep = (new TileIO) { new ProbeReply } - val probe_rep_data = (new TileIO) { new ProbeReplyData } - val xact_rep = (new HubIO) { new TransactionReply } - val xact_finish = (new TileIO) { new TransactionFinish } - val incoherent = Bool(OUTPUT) -} - class PhysicalAddress extends Bundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) } From 400d48e3de0a00444cef01e03829cfbb2450ced8 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 13 Dec 2012 11:39:14 -0800 Subject: [PATCH 152/374] Refactored uncore conf --- uncore/src/coherence.scala | 4 ++ uncore/src/package.scala | 3 +- uncore/src/uncore.scala | 124 ++++++++++++++++++------------------- 3 files changed, 68 insertions(+), 63 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index e8aad9d0..96fd475d 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -3,6 +3,10 @@ package uncore import Chisel._ import Constants._ +trait CoherenceAgent +trait ClientCoherenceAgent extends CoherenceAgent +trait MasterCoherenceAgent extends CoherenceAgent + object cpuCmdToRW { def apply(cmd: Bits): (Bool, Bool) = (isRead(cmd) || isPrefetch(cmd), isWrite(cmd)) } diff --git a/uncore/src/package.scala b/uncore/src/package.scala index a3af64e1..137cc74d 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -35,7 +35,7 @@ class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfi case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nHubs: Int, nTiles: Int) -abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[Component])(implicit conf: LogicalNetworkConfiguration) extends Component { +abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[CoherenceAgent])(implicit conf: LogicalNetworkConfiguration) extends Component { val io: Vec[TileLinkType] val physicalNetworks: Seq[PhysicalNetwork] require(endpoints.length == conf.nEndpoints) @@ -63,5 +63,6 @@ class TileLink(implicit conf: LogicalNetworkConfiguration) extends Bundle { val xact_rep = (new HubIO) { new TransactionReply } val xact_finish = (new TileIO) { new TransactionFinish } val incoherent = Bool(OUTPUT) + override def clone = { new TileLink().asInstanceOf[this.type] } } } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index e2dcfc30..99136c1b 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -3,13 +3,13 @@ package uncore import Chisel._ import Constants._ -class TrackerProbeData(implicit conf: UncoreConfiguration) extends Bundle { - val tile_id = Bits(width = conf.tile_id_bits) +class TrackerProbeData(implicit conf: CoherenceHubConfiguration) extends Bundle { + val tile_id = Bits(width = conf.ln.idBits) } -class TrackerAllocReq(implicit conf: UncoreConfiguration) extends Bundle { +class TrackerAllocReq(implicit conf: CoherenceHubConfiguration) extends Bundle { val xact_init = new TransactionInit() - val tile_id = Bits(width = conf.tile_id_bits) + val tile_id = Bits(width = conf.ln.idBits) override def clone = { new TrackerAllocReq().asInstanceOf[this.type] } } @@ -17,16 +17,16 @@ class TrackerDependency extends Bundle { val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } -class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component { +class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Component { val co = conf.co val io = new Bundle { val alloc_req = (new FIFOIO) { new TrackerAllocReq }.flip val p_data = (new PipeIO) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) - val p_rep_cnt_dec = Bits(INPUT, conf.ntiles) - val p_req_cnt_inc = Bits(INPUT, conf.ntiles) - val tile_incoherent = Bits(INPUT, conf.ntiles) + val p_rep_cnt_dec = Bits(INPUT, conf.ln.nTiles) + val p_req_cnt_inc = Bits(INPUT, conf.ln.nTiles) + val tile_incoherent = Bits(INPUT, conf.ln.nTiles) val p_rep_data = (new PipeIO) { new ProbeReplyData }.flip val x_init_data = (new PipeIO) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) @@ -39,18 +39,18 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component val probe_req = (new FIFOIO) { new ProbeRequest } val busy = Bool(OUTPUT) val addr = Bits(OUTPUT, PADDR_BITS - OFFSET_BITS) - val init_tile_id = Bits(OUTPUT, conf.tile_id_bits) - val p_rep_tile_id = Bits(OUTPUT, conf.tile_id_bits) + val init_tile_id = Bits(OUTPUT, conf.ln.idBits) + val p_rep_tile_id = Bits(OUTPUT, conf.ln.idBits) val tile_xact_id = Bits(OUTPUT, TILE_XACT_ID_BITS) - val sharer_count = Bits(OUTPUT, conf.tile_id_bits+1) + val sharer_count = Bits(OUTPUT, conf.ln.idBits+1) val x_type = Bits(OUTPUT, X_INIT_TYPE_MAX_BITS) - val push_p_req = Bits(OUTPUT, conf.ntiles) - val pop_p_rep = Bits(OUTPUT, conf.ntiles) - val pop_p_rep_data = Bits(OUTPUT, conf.ntiles) - val pop_p_rep_dep = Bits(OUTPUT, conf.ntiles) - val pop_x_init = Bits(OUTPUT, conf.ntiles) - val pop_x_init_data = Bits(OUTPUT, conf.ntiles) - val pop_x_init_dep = Bits(OUTPUT, conf.ntiles) + val push_p_req = Bits(OUTPUT, conf.ln.nTiles) + val pop_p_rep = Bits(OUTPUT, conf.ln.nTiles) + val pop_p_rep_data = Bits(OUTPUT, conf.ln.nTiles) + val pop_p_rep_dep = Bits(OUTPUT, conf.ln.nTiles) + val pop_x_init = Bits(OUTPUT, conf.ln.nTiles) + val pop_x_init_data = Bits(OUTPUT, conf.ln.nTiles) + val pop_x_init_dep = Bits(OUTPUT, conf.ln.nTiles) val send_x_rep_ack = Bool(OUTPUT) } @@ -91,8 +91,8 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component val state = Reg(resetVal = s_idle) val xact = Reg{ new TransactionInit } val init_tile_id_ = Reg{ Bits() } - val p_rep_count = if (conf.ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ntiles))) - val p_req_flags = Reg(resetVal = Bits(0, width = conf.ntiles)) + val p_rep_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val p_req_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) val p_rep_tile_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) val x_init_data_needs_write = Reg(resetVal = Bool(false)) @@ -102,16 +102,16 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) - val p_req_initial_flags = Bits(width = conf.ntiles) + val p_req_initial_flags = Bits(width = conf.ln.nTiles) p_req_initial_flags := Bits(0) - if (conf.ntiles > 1) { + if (conf.ln.nTiles > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence // TODO: this is hackish; figure out how to do it more systematically val probe_self = co match { case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.alloc_req.bits.xact_init) case _ => Bool(false) } - val myflag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.tile_id(log2Up(conf.ntiles)-1,0))) + val myflag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.tile_id(log2Up(conf.ln.nTiles)-1,0))) p_req_initial_flags := ~(io.tile_incoherent | myflag) } @@ -120,7 +120,7 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component io.init_tile_id := init_tile_id_ io.p_rep_tile_id := p_rep_tile_id_ io.tile_xact_id := xact.tile_xact_id - io.sharer_count := UFix(conf.ntiles) // TODO: Broadcast only + io.sharer_count := UFix(conf.ln.nTiles) // TODO: Broadcast only io.x_type := xact.x_type io.mem_req_cmd.valid := Bool(false) @@ -134,13 +134,13 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component io.probe_req.bits.p_type := co.getProbeRequestType(xact.x_type, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) io.probe_req.bits.addr := xact.addr - io.push_p_req := Bits(0, width = conf.ntiles) - io.pop_p_rep := Bits(0, width = conf.ntiles) - io.pop_p_rep_data := Bits(0, width = conf.ntiles) - io.pop_p_rep_dep := Bits(0, width = conf.ntiles) - io.pop_x_init := Bits(0, width = conf.ntiles) - io.pop_x_init_data := Bits(0, width = conf.ntiles) - io.pop_x_init_dep := Bits(0, width = conf.ntiles) + io.push_p_req := Bits(0, width = conf.ln.nTiles) + io.pop_p_rep := Bits(0, width = conf.ln.nTiles) + io.pop_p_rep_data := Bits(0, width = conf.ln.nTiles) + io.pop_p_rep_dep := Bits(0, width = conf.ln.nTiles) + io.pop_x_init := Bits(0, width = conf.ln.nTiles) + io.pop_x_init_data := Bits(0, width = conf.ln.nTiles) + io.pop_x_init_dep := Bits(0, width = conf.ln.nTiles) io.send_x_rep_ack := Bool(false) switch (state) { @@ -155,7 +155,7 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id - if(conf.ntiles > 1) { + if(conf.ln.nTiles > 1) { p_rep_count := PopCount(p_req_initial_flags) state := Mux(p_req_initial_flags.orR, s_probe, s_mem) } else state := s_mem @@ -172,7 +172,7 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component when(io.p_rep_cnt_dec.orR) { val dec = PopCount(io.p_rep_cnt_dec) io.pop_p_rep := io.p_rep_cnt_dec - if(conf.ntiles > 1) p_rep_count := p_rep_count - dec + if(conf.ln.nTiles > 1) p_rep_count := p_rep_count - dec when(p_rep_count === dec) { state := s_mem } @@ -223,16 +223,16 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component } } -case class UncoreConfiguration(ntiles: Int, tile_id_bits: Int, co: CoherencePolicy) +case class CoherenceHubConfiguration(co: CoherencePolicy, ln: LogicalNetworkConfiguration) -abstract class CoherenceHub(implicit conf: UncoreConfiguration) extends Component { +abstract class CoherenceHub(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { val io = new Bundle { - val tiles = Vec(conf.ntiles) { new ioTileLink }.flip + val tiles = Vec(conf.nTiles) { new TileLink }.flip val mem = new ioMem } } -class CoherenceHubNull(implicit conf: UncoreConfiguration) extends CoherenceHub +class CoherenceHubNull(implicit conf: CoherenceHubConfiguration) extends CoherenceHub()(conf.ln) { val co = conf.co.asInstanceOf[ThreeStateIncoherence] @@ -261,24 +261,24 @@ class CoherenceHubNull(implicit conf: UncoreConfiguration) extends CoherenceHub } -class CoherenceHubBroadcast(implicit conf: UncoreConfiguration) extends CoherenceHub +class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends CoherenceHub()(conf.ln) { val co = conf.co val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) val busy_arr = Vec(NGLOBAL_XACTS){ Bool() } val addr_arr = Vec(NGLOBAL_XACTS){ Bits(width=PADDR_BITS-OFFSET_BITS) } - val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.tile_id_bits) } + val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_XACT_ID_BITS) } val x_type_arr = Vec(NGLOBAL_XACTS){ Bits(width=X_INIT_TYPE_MAX_BITS) } - val sh_count_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.tile_id_bits) } + val sh_count_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } val do_free_arr = Vec(NGLOBAL_XACTS){ Bool() } - val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ntiles){ Bool()} } - val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ntiles){ Bool()} } + val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } + val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } - val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.tile_id_bits) } + val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } for( i <- 0 until NGLOBAL_XACTS) { @@ -299,19 +299,19 @@ class CoherenceHubBroadcast(implicit conf: UncoreConfiguration) extends Coherenc t.sent_x_rep_ack := sent_x_rep_ack_arr(i) do_free_arr(i) := Bool(false) sent_x_rep_ack_arr(i) := Bool(false) - p_data_tile_id_arr(i) := Bits(0, width = conf.tile_id_bits) + p_data_tile_id_arr(i) := Bits(0, width = conf.ln.idBits) p_data_valid_arr(i) := Bool(false) - for( j <- 0 until conf.ntiles) { + for( j <- 0 until conf.ln.nTiles) { p_rep_cnt_dec_arr(i)(j) := Bool(false) p_req_cnt_inc_arr(i)(j) := Bool(false) } } - val p_rep_data_dep_list = List.fill(conf.ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY - val x_init_data_dep_list = List.fill(conf.ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY + val p_rep_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY + val x_init_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY // Free finished transactions - for( j <- 0 until conf.ntiles ) { + for( j <- 0 until conf.ln.nTiles ) { val finish = io.tiles(j).xact_finish when (finish.valid) { do_free_arr(finish.bits.global_xact_id) := Bool(true) @@ -323,7 +323,7 @@ class CoherenceHubBroadcast(implicit conf: UncoreConfiguration) extends Coherenc // Forward memory responses from mem to tile or arbitrate to ack val mem_idx = io.mem.resp.bits.tag val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) - for( j <- 0 until conf.ntiles ) { + for( j <- 0 until conf.ln.nTiles ) { val rep = io.tiles(j).xact_rep rep.bits.x_type := UFix(0) rep.bits.tile_xact_id := UFix(0) @@ -362,7 +362,7 @@ class CoherenceHubBroadcast(implicit conf: UncoreConfiguration) extends Coherenc io.mem.req_data <> Queue(mem_req_data_arb.io.out) // Handle probe replies, which may or may not have data - for( j <- 0 until conf.ntiles ) { + for( j <- 0 until conf.ln.nTiles ) { val p_rep = io.tiles(j).probe_rep val p_rep_data = io.tiles(j).probe_rep_data val idx = p_rep.bits.global_xact_id @@ -382,10 +382,10 @@ class CoherenceHubBroadcast(implicit conf: UncoreConfiguration) extends Coherenc trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits - trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until conf.ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) - trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until conf.ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) + trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) + trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) - for( j <- 0 until conf.ntiles) { + for( j <- 0 until conf.ln.nTiles) { val p_rep = io.tiles(j).probe_rep p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) } @@ -393,9 +393,9 @@ class CoherenceHubBroadcast(implicit conf: UncoreConfiguration) extends Coherenc // Nack conflicting transaction init attempts val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } - val abort_state_arr = Vec(conf.ntiles) { Reg(resetVal = s_idle) } - val want_to_abort_arr = Vec(conf.ntiles) { Bool() } - for( j <- 0 until conf.ntiles ) { + val abort_state_arr = Vec(conf.ln.nTiles) { Reg(resetVal = s_idle) } + val want_to_abort_arr = Vec(conf.ln.nTiles) { Bool() } + for( j <- 0 until conf.ln.nTiles ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_abort = io.tiles(j).xact_abort @@ -440,7 +440,7 @@ class CoherenceHubBroadcast(implicit conf: UncoreConfiguration) extends Coherenc // Only one allocation per cycle // Init requests may or may not have data val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } - val init_arb = (new Arbiter(conf.ntiles)) { new TrackerAllocReq } + val init_arb = (new Arbiter(conf.ln.nTiles)) { new TrackerAllocReq } for( i <- 0 until NGLOBAL_XACTS ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready @@ -449,10 +449,10 @@ class CoherenceHubBroadcast(implicit conf: UncoreConfiguration) extends Coherenc trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid - trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until conf.ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) - trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until conf.ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) + trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) + trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) } - for( j <- 0 until conf.ntiles ) { + for( j <- 0 until conf.ln.nTiles ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_init_data_dep = x_init_data_dep_list(j).io.deq @@ -473,8 +473,8 @@ class CoherenceHubBroadcast(implicit conf: UncoreConfiguration) extends Coherenc // Handle probe request generation // Must arbitrate for each request port - val p_req_arb_arr = List.fill(conf.ntiles)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) - for( j <- 0 until conf.ntiles ) { + val p_req_arb_arr = List.fill(conf.ln.nTiles)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) + for( j <- 0 until conf.ln.nTiles ) { for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits From 418e3fdf5046111bc34edb00462802ebba20ddc9 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 7 Jan 2013 13:57:48 -0800 Subject: [PATCH 153/374] Bugfix in crossbar ready sigs, added adapter for old hub, removed incoherent sig from tilelink --- uncore/src/package.scala | 29 +++++++++++++++++---------- uncore/src/uncore.scala | 42 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 59 insertions(+), 12 deletions(-) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 137cc74d..9698ffc3 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -1,6 +1,9 @@ package object uncore { import Chisel._ +//TODO: Remove these Networking classes from the package object once Scala bug +//SI-3439 is resolved. + case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { @@ -22,14 +25,21 @@ class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfi val out = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } } } - for(i <- 0 until conf.nEndpoints) { + val rdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) + + io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { val rrarb = new RRArbiter(conf.nEndpoints)(data) - (rrarb.io.in, io.in).zipped.map( (arb, io) => { - arb.valid := io.valid && (io.header.dst === UFix(i)) - arb.bits := io.bits - io.ready := arb.ready - }) - io.out(i) <> rrarb.io.out + (rrarb.io.in, io.in, rdys).zipped.map{ case (arb, in, rdy) => { + arb.valid := in.valid && (in.header.dst === UFix(i)) + arb.bits := in.bits + rdy := arb.ready && (in.header.dst === UFix(i)) + }} + out <> rrarb.io.out + out.header.src := rrarb.io.chosen.toUFix + out.header.dst := UFix(i) + }} + for(i <- 0 until conf.nEndpoints) { + io.in(i).ready := rdyVecs.map(r => r(i)).reduceLeft(_||_) } } @@ -53,7 +63,7 @@ abstract class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalN class TileIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(conf) class HubIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(conf){flip()} -class TileLink(implicit conf: LogicalNetworkConfiguration) extends Bundle { +class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { val xact_init = (new TileIO) { new TransactionInit } val xact_init_data = (new TileIO) { new TransactionInitData } val xact_abort = (new HubIO) { new TransactionAbort } @@ -62,7 +72,6 @@ class TileLink(implicit conf: LogicalNetworkConfiguration) extends Bundle { val probe_rep_data = (new TileIO) { new ProbeReplyData } val xact_rep = (new HubIO) { new TransactionReply } val xact_finish = (new TileIO) { new TransactionFinish } - val incoherent = Bool(OUTPUT) - override def clone = { new TileLink().asInstanceOf[this.type] } + override def clone = { new TileLinkIO().asInstanceOf[this.type] } } } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 99136c1b..daf756f0 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -225,9 +225,47 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com case class CoherenceHubConfiguration(co: CoherencePolicy, ln: LogicalNetworkConfiguration) +class CoherenceHubAdapter(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { + val io = new Bundle { + val net = (new TileLinkIO).flip + val hub = Vec(conf.nTiles) { new TileLinkIO } + } + + val netTileProducedSubBundles = io.net.getClass.getMethods.filter( x => + classOf[TileIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => + m.invoke(io.net).asInstanceOf[TileIO[Data]] } + val netHubProducedSubBundles = io.net.getClass.getMethods.filter( x => + classOf[HubIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => + m.invoke(io.net).asInstanceOf[HubIO[Data]] } + + val hubTileProducedSubBundles = io.hub.map{ io => { + io.getClass.getMethods.filter( x => + classOf[TileIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => + m.invoke(io).asInstanceOf[TileIO[Data]] }}}.transpose + val hubHubProducedSubBundles = io.hub.map{ io => { + io.getClass.getMethods.filter( x => + classOf[HubIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => + m.invoke(io).asInstanceOf[HubIO[Data]] }}}.transpose + + hubHubProducedSubBundles.zip(netHubProducedSubBundles).foreach{ case(hub, net) => { + net.header.src := UFix(0) + net.header.dst := Vec(hub.map(_.valid)){Bool()}.indexWhere{s: Bool => s} + net.valid := hub.map(_.valid).fold(Bool(false))(_||_) + net.bits := hub(0).bits + hub.foreach( _.ready := net.ready) + }} + hubTileProducedSubBundles.zip(netTileProducedSubBundles).foreach{ case(hub, net) => { + hub.foreach(_.header := net.header) + hub.zipWithIndex.foreach{ case(h,i) => h.valid := (net.header.src === UFix(i)) && net.valid } + hub.foreach(_.bits := net.bits) + net.ready := hub.map(_.ready).fold(Bool(false))(_||_) + }} +} + abstract class CoherenceHub(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { val io = new Bundle { - val tiles = Vec(conf.nTiles) { new TileLink }.flip + val tiles = Vec(conf.nTiles) { new TileLinkIO }.flip + val incoherent = Vec(conf.nTiles) { Bool() }.asInput val mem = new ioMem } } @@ -295,7 +333,7 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co t.p_data.valid := p_data_valid_arr(i) t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits - t.tile_incoherent := (Vec(io.tiles.map(_.incoherent)) { Bool() }).toBits + t.tile_incoherent := io.incoherent.toBits t.sent_x_rep_ack := sent_x_rep_ack_arr(i) do_free_arr(i) := Bool(false) sent_x_rep_ack_arr(i) := Bool(false) From f7c01524099f1880ba395e15f9474c50b9d8272c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 15 Jan 2013 15:52:47 -0800 Subject: [PATCH 154/374] Refactored packet headers/payloads --- uncore/src/coherence.scala | 6 +- uncore/src/package.scala | 92 ++++++++++++++----- uncore/src/tilelink.scala | 2 + uncore/src/uncore.scala | 179 +++++++++++++++++++------------------ 4 files changed, 164 insertions(+), 115 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index 96fd475d..caa2b648 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -3,9 +3,9 @@ package uncore import Chisel._ import Constants._ -trait CoherenceAgent -trait ClientCoherenceAgent extends CoherenceAgent -trait MasterCoherenceAgent extends CoherenceAgent +trait CoherenceAgentRole +trait ClientCoherenceAgent extends CoherenceAgentRole +trait MasterCoherenceAgent extends CoherenceAgentRole object cpuCmdToRW { def apply(cmd: Bits): (Bool, Bool) = (isRead(cmd) || isPrefetch(cmd), isWrite(cmd)) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 9698ffc3..3c2a93df 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -1,5 +1,7 @@ package object uncore { import Chisel._ +import Node._ +import scala.collection.mutable.Stack //TODO: Remove these Networking classes from the package object once Scala bug //SI-3439 is resolved. @@ -11,32 +13,35 @@ class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle val dst = UFix(width = conf.idBits) } -abstract class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends FIFOIO()(data) { - val header = (new PhysicalHeader).asOutput +abstract class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { + val header = (new PhysicalHeader) + val payload = data } -class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetworkIO()(data)(conf) +class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetworkIO()(data)(conf) { + override def clone = { new BasicCrossbarIO()(data).asInstanceOf[this.type] } +} abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Component class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { val io = new Bundle { - val in = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } }.flip - val out = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } } + val in = Vec(conf.nEndpoints){(new FIFOIO){(new BasicCrossbarIO){data}}}.flip + val out = Vec(conf.nEndpoints){(new FIFOIO){(new BasicCrossbarIO){data}}} } val rdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { - val rrarb = new RRArbiter(conf.nEndpoints)(data) + val rrarb = (new RRArbiter(conf.nEndpoints)){io.in(0).bits.clone} (rrarb.io.in, io.in, rdys).zipped.map{ case (arb, in, rdy) => { - arb.valid := in.valid && (in.header.dst === UFix(i)) + arb.valid := in.valid && (in.bits.header.dst === UFix(i)) arb.bits := in.bits - rdy := arb.ready && (in.header.dst === UFix(i)) + rdy := arb.ready && (in.bits.header.dst === UFix(i)) }} out <> rrarb.io.out - out.header.src := rrarb.io.chosen.toUFix - out.header.dst := UFix(i) + //out.bits.header.src := rrarb.io.chosen.toUFix + //out.bits.header.dst := UFix(i) }} for(i <- 0 until conf.nEndpoints) { io.in(i).ready := rdyVecs.map(r => r(i)).reduceLeft(_||_) @@ -45,7 +50,7 @@ class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfi case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nHubs: Int, nTiles: Int) -abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[CoherenceAgent])(implicit conf: LogicalNetworkConfiguration) extends Component { +abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[CoherenceAgentRole])(implicit conf: LogicalNetworkConfiguration) extends Component { val io: Vec[TileLinkType] val physicalNetworks: Seq[PhysicalNetwork] require(endpoints.length == conf.nEndpoints) @@ -56,22 +61,63 @@ class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { val dst = UFix(width = conf.idBits) } -abstract class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends FIFOIO()(data) { - val header = (new LogicalHeader).asOutput +object FIFOedLogicalNetworkIOWrapper { + def apply[T <: Data](in: FIFOIO[T])(implicit conf: LogicalNetworkConfiguration) = { + val shim = (new FIFOedLogicalNetworkIOWrapper){ in.bits.clone } + shim.io.in.valid := in.valid + shim.io.in.bits := in.bits + in.ready := shim.io.in.ready + shim.io.out + } +} +class FIFOedLogicalNetworkIOWrapper[T <: Data]()(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { + val io = new Bundle { + val in = (new FIFOIO){ data }.flip + val out = (new FIFOIO){(new LogicalNetworkIO){ data }} + } + io.out.valid := io.in.valid + io.out.bits.payload := io.in.bits + io.in.ready := io.out.ready } -class TileIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(conf) -class HubIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(conf){flip()} +object FIFOedLogicalNetworkIOUnwrapper { + def apply[T <: Data](in: FIFOIO[LogicalNetworkIO[T]])(implicit conf: LogicalNetworkConfiguration) = { + val shim = (new FIFOedLogicalNetworkIOUnwrapper){ in.bits.payload.clone } + shim.io.in.valid := in.valid + shim.io.in.bits := in.bits + in.ready := shim.io.in.ready + shim.io.out + } +} +class FIFOedLogicalNetworkIOUnwrapper[T <: Data]()(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { + val io = new Bundle { + val in = (new FIFOIO){(new LogicalNetworkIO){ data }}.flip + val out = (new FIFOIO){ data } + } + io.out.valid := io.in.valid + io.out.bits := io.in.bits.payload + io.in.ready := io.out.ready +} + +class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val header = new LogicalHeader + val payload = data + override def clone = { new LogicalNetworkIO()(data).asInstanceOf[this.type] } +} + +abstract class DirectionalFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) +class ClientSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) +class MasterSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) {flip()} class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val xact_init = (new TileIO) { new TransactionInit } - val xact_init_data = (new TileIO) { new TransactionInitData } - val xact_abort = (new HubIO) { new TransactionAbort } - val probe_req = (new HubIO) { new ProbeRequest } - val probe_rep = (new TileIO) { new ProbeReply } - val probe_rep_data = (new TileIO) { new ProbeReplyData } - val xact_rep = (new HubIO) { new TransactionReply } - val xact_finish = (new TileIO) { new TransactionFinish } + val xact_init = (new ClientSourcedIO){(new LogicalNetworkIO){new TransactionInit }} + val xact_init_data = (new ClientSourcedIO){(new LogicalNetworkIO){new TransactionInitData }} + val xact_abort = (new MasterSourcedIO) {(new LogicalNetworkIO){new TransactionAbort }} + val probe_req = (new MasterSourcedIO) {(new LogicalNetworkIO){new ProbeRequest }} + val probe_rep = (new ClientSourcedIO){(new LogicalNetworkIO){new ProbeReply }} + val probe_rep_data = (new ClientSourcedIO){(new LogicalNetworkIO){new ProbeReplyData }} + val xact_rep = (new MasterSourcedIO) {(new LogicalNetworkIO){new TransactionReply }} + val xact_finish = (new ClientSourcedIO){(new LogicalNetworkIO){new TransactionFinish }} override def clone = { new TileLinkIO().asInstanceOf[this.type] } } } diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index a9cbc482..f07e7ed1 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -97,6 +97,7 @@ class TransactionFinish extends Bundle { val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } +/* class ioTileLink extends Bundle { val xact_init = (new FIFOIO) { new TransactionInit } val xact_init_data = (new FIFOIO) { new TransactionInitData } @@ -108,3 +109,4 @@ class ioTileLink extends Bundle { val xact_finish = (new FIFOIO) { new TransactionFinish } val incoherent = Bool(OUTPUT) } +*/ diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index daf756f0..697b632c 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -4,12 +4,12 @@ import Chisel._ import Constants._ class TrackerProbeData(implicit conf: CoherenceHubConfiguration) extends Bundle { - val tile_id = Bits(width = conf.ln.idBits) + val client_id = Bits(width = conf.ln.idBits) } class TrackerAllocReq(implicit conf: CoherenceHubConfiguration) extends Bundle { val xact_init = new TransactionInit() - val tile_id = Bits(width = conf.ln.idBits) + val client_id = Bits(width = conf.ln.idBits) override def clone = { new TrackerAllocReq().asInstanceOf[this.type] } } @@ -17,7 +17,7 @@ class TrackerDependency extends Bundle { val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } -class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Component { +class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) extends Component { val co = conf.co val io = new Bundle { val alloc_req = (new FIFOIO) { new TrackerAllocReq }.flip @@ -39,8 +39,8 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val probe_req = (new FIFOIO) { new ProbeRequest } val busy = Bool(OUTPUT) val addr = Bits(OUTPUT, PADDR_BITS - OFFSET_BITS) - val init_tile_id = Bits(OUTPUT, conf.ln.idBits) - val p_rep_tile_id = Bits(OUTPUT, conf.ln.idBits) + val init_client_id = Bits(OUTPUT, conf.ln.idBits) + val p_rep_client_id = Bits(OUTPUT, conf.ln.idBits) val tile_xact_id = Bits(OUTPUT, TILE_XACT_ID_BITS) val sharer_count = Bits(OUTPUT, conf.ln.idBits+1) val x_type = Bits(OUTPUT, X_INIT_TYPE_MAX_BITS) @@ -54,7 +54,7 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val send_x_rep_ack = Bool(OUTPUT) } - def doMemReqWrite(req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: PipeIO[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { + def doMemReqWrite(req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: PipeIO[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, client_id: UFix) { req_cmd.bits.rw := Bool(true) req_data.bits := data.bits when(req_cmd.ready && req_cmd.valid) { @@ -66,11 +66,11 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com when (req_cmd.ready || cmd_sent) { req_data.valid := data.valid when(req_data.ready) { - pop_data := UFix(1) << tile_id + pop_data := UFix(1) << client_id when (data.valid) { mem_cnt := mem_cnt_next when(mem_cnt === UFix(REFILL_CYCLES-1)) { - pop_dep := UFix(1) << tile_id + pop_dep := UFix(1) << client_id trigger := Bool(false) } } @@ -90,10 +90,10 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) val xact = Reg{ new TransactionInit } - val init_tile_id_ = Reg{ Bits() } + val init_client_id_ = Reg{ Bits() } val p_rep_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) val p_req_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) - val p_rep_tile_id_ = Reg{ Bits() } + val p_rep_client_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) val x_init_data_needs_write = Reg(resetVal = Bool(false)) val p_rep_data_needs_write = Reg(resetVal = Bool(false)) @@ -111,14 +111,14 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.alloc_req.bits.xact_init) case _ => Bool(false) } - val myflag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.tile_id(log2Up(conf.ln.nTiles)-1,0))) + val myflag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.client_id(log2Up(conf.ln.nTiles)-1,0))) p_req_initial_flags := ~(io.tile_incoherent | myflag) } io.busy := state != s_idle io.addr := xact.addr - io.init_tile_id := init_tile_id_ - io.p_rep_tile_id := p_rep_tile_id_ + io.init_client_id := init_client_id_ + io.p_rep_client_id := p_rep_client_id_ io.tile_xact_id := xact.tile_xact_id io.sharer_count := UFix(conf.ln.nTiles) // TODO: Broadcast only io.x_type := xact.x_type @@ -147,14 +147,14 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com is(s_idle) { when( io.alloc_req.valid && io.can_alloc ) { xact := io.alloc_req.bits.xact_init - init_tile_id_ := io.alloc_req.bits.tile_id + init_client_id_ := io.alloc_req.bits.client_id x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) p_req_flags := p_req_initial_flags mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) - io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id + io.pop_x_init := UFix(1) << io.alloc_req.bits.client_id if(conf.ln.nTiles > 1) { p_rep_count := PopCount(p_req_initial_flags) state := Mux(p_req_initial_flags.orR, s_probe, s_mem) @@ -179,7 +179,7 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com } when(io.p_data.valid) { p_rep_data_needs_write := Bool(true) - p_rep_tile_id_ := io.p_data.bits.tile_id + p_rep_client_id_ := io.p_data.bits.client_id } } is(s_mem) { @@ -193,7 +193,7 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com io.pop_p_rep_data, io.pop_p_rep_dep, io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id)), - p_rep_tile_id_) + p_rep_client_id_) } . elsewhen(x_init_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, @@ -204,7 +204,7 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com io.pop_x_init_data, io.pop_x_init_dep, io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id)), - init_tile_id_) + init_client_id_) } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { @@ -231,33 +231,33 @@ class CoherenceHubAdapter(implicit conf: LogicalNetworkConfiguration) extends Co val hub = Vec(conf.nTiles) { new TileLinkIO } } - val netTileProducedSubBundles = io.net.getClass.getMethods.filter( x => - classOf[TileIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => - m.invoke(io.net).asInstanceOf[TileIO[Data]] } - val netHubProducedSubBundles = io.net.getClass.getMethods.filter( x => - classOf[HubIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => - m.invoke(io.net).asInstanceOf[HubIO[Data]] } + val netClientProducedSubBundles = io.net.getClass.getMethods.filter( x => + classOf[ClientSourcedIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => + m.invoke(io.net).asInstanceOf[ClientSourcedIO[LogicalNetworkIO[Data]]] } + val netMasterProducedSubBundles = io.net.getClass.getMethods.filter( x => + classOf[MasterSourcedIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => + m.invoke(io.net).asInstanceOf[MasterSourcedIO[LogicalNetworkIO[Data]]] } - val hubTileProducedSubBundles = io.hub.map{ io => { + val hubClientProducedSubBundles = io.hub.map{ io => { io.getClass.getMethods.filter( x => - classOf[TileIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => - m.invoke(io).asInstanceOf[TileIO[Data]] }}}.transpose - val hubHubProducedSubBundles = io.hub.map{ io => { + classOf[ClientSourcedIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => + m.invoke(io).asInstanceOf[ClientSourcedIO[LogicalNetworkIO[Data]]] }}}.transpose + val hubMasterProducedSubBundles = io.hub.map{ io => { io.getClass.getMethods.filter( x => - classOf[HubIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => - m.invoke(io).asInstanceOf[HubIO[Data]] }}}.transpose + classOf[MasterSourcedIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => + m.invoke(io).asInstanceOf[MasterSourcedIO[LogicalNetworkIO[Data]]] }}}.transpose - hubHubProducedSubBundles.zip(netHubProducedSubBundles).foreach{ case(hub, net) => { - net.header.src := UFix(0) - net.header.dst := Vec(hub.map(_.valid)){Bool()}.indexWhere{s: Bool => s} + hubMasterProducedSubBundles.zip(netMasterProducedSubBundles).foreach{ case(hub, net) => { + net.bits.header.src := UFix(0) + net.bits.header.dst := Vec(hub.map(_.valid)){Bool()}.indexWhere{s: Bool => s} + net.bits.payload := hub(0).bits.payload net.valid := hub.map(_.valid).fold(Bool(false))(_||_) - net.bits := hub(0).bits hub.foreach( _.ready := net.ready) }} - hubTileProducedSubBundles.zip(netTileProducedSubBundles).foreach{ case(hub, net) => { - hub.foreach(_.header := net.header) - hub.zipWithIndex.foreach{ case(h,i) => h.valid := (net.header.src === UFix(i)) && net.valid } - hub.foreach(_.bits := net.bits) + hubClientProducedSubBundles.zip(netClientProducedSubBundles).foreach{ case(hub, net) => { + hub.foreach(_.bits.header := net.bits.header) + hub.zipWithIndex.foreach{ case(h,i) => h.valid := (net.bits.header.src === UFix(i)) && net.valid } + hub.foreach(_.bits.payload := net.bits.payload) net.ready := hub.map(_.ready).fold(Bool(false))(_||_) }} } @@ -275,20 +275,20 @@ class CoherenceHubNull(implicit conf: CoherenceHubConfiguration) extends Coheren val co = conf.co.asInstanceOf[ThreeStateIncoherence] val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.x_type === co.xactInitWriteback + val is_write = x_init.bits.payload.x_type === co.xactInitWriteback x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write - io.mem.req_cmd.bits.tag := x_init.bits.tile_xact_id - io.mem.req_cmd.bits.addr := x_init.bits.addr + io.mem.req_cmd.bits.tag := x_init.bits.payload.tile_xact_id + io.mem.req_cmd.bits.addr := x_init.bits.payload.addr io.mem.req_data <> io.tiles(0).xact_init_data val x_rep = io.tiles(0).xact_rep - x_rep.bits.x_type := Mux(io.mem.resp.valid, co.xactReplyData, co.xactReplyAck) - x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) - x_rep.bits.global_xact_id := UFix(0) // don't care - x_rep.bits.data := io.mem.resp.bits.data - x_rep.bits.require_ack := Bool(true) + x_rep.bits.payload.x_type := Mux(io.mem.resp.valid, co.xactReplyData, co.xactReplyAck) + x_rep.bits.payload.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.payload.tile_xact_id) + x_rep.bits.payload.global_xact_id := UFix(0) // don't care + x_rep.bits.payload.data := io.mem.resp.bits.data + x_rep.bits.payload.require_ack := Bool(true) x_rep.valid := io.mem.resp.valid || x_init.valid && is_write && io.mem.req_cmd.ready io.tiles(0).xact_abort.valid := Bool(false) @@ -301,12 +301,13 @@ class CoherenceHubNull(implicit conf: CoherenceHubConfiguration) extends Coheren class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends CoherenceHub()(conf.ln) { + implicit val lnConf = conf.ln val co = conf.co - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTrackerBroadcast(_)) val busy_arr = Vec(NGLOBAL_XACTS){ Bool() } val addr_arr = Vec(NGLOBAL_XACTS){ Bits(width=PADDR_BITS-OFFSET_BITS) } - val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } + val init_client_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_XACT_ID_BITS) } val x_type_arr = Vec(NGLOBAL_XACTS){ Bits(width=X_INIT_TYPE_MAX_BITS) } val sh_count_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } @@ -316,20 +317,20 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } - val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } + val p_data_client_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io busy_arr(i) := t.busy addr_arr(i) := t.addr - init_tile_id_arr(i) := t.init_tile_id + init_client_id_arr(i) := t.init_client_id tile_xact_id_arr(i) := t.tile_xact_id x_type_arr(i) := t.x_type sh_count_arr(i) := t.sharer_count send_x_rep_ack_arr(i) := t.send_x_rep_ack t.xact_finish := do_free_arr(i) - t.p_data.bits.tile_id := p_data_tile_id_arr(i) + t.p_data.bits.client_id := p_data_client_id_arr(i) t.p_data.valid := p_data_valid_arr(i) t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits @@ -337,7 +338,7 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co t.sent_x_rep_ack := sent_x_rep_ack_arr(i) do_free_arr(i) := Bool(false) sent_x_rep_ack_arr(i) := Bool(false) - p_data_tile_id_arr(i) := Bits(0, width = conf.ln.idBits) + p_data_client_id_arr(i) := Bits(0, width = conf.ln.idBits) p_data_valid_arr(i) := Bool(false) for( j <- 0 until conf.ln.nTiles) { p_rep_cnt_dec_arr(i)(j) := Bool(false) @@ -352,7 +353,7 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co for( j <- 0 until conf.ln.nTiles ) { val finish = io.tiles(j).xact_finish when (finish.valid) { - do_free_arr(finish.bits.global_xact_id) := Bool(true) + do_free_arr(finish.bits.payload.global_xact_id) := Bool(true) } finish.ready := Bool(true) } @@ -363,28 +364,28 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) for( j <- 0 until conf.ln.nTiles ) { val rep = io.tiles(j).xact_rep - rep.bits.x_type := UFix(0) - rep.bits.tile_xact_id := UFix(0) - rep.bits.global_xact_id := UFix(0) - rep.bits.data := io.mem.resp.bits.data - rep.bits.require_ack := Bool(true) + rep.bits.payload.x_type := UFix(0) + rep.bits.payload.tile_xact_id := UFix(0) + rep.bits.payload.global_xact_id := UFix(0) + rep.bits.payload.data := io.mem.resp.bits.data + rep.bits.payload.require_ack := Bool(true) rep.valid := Bool(false) - when(io.mem.resp.valid && (UFix(j) === init_tile_id_arr(mem_idx))) { - rep.bits.x_type := co.getTransactionReplyType(x_type_arr(mem_idx), sh_count_arr(mem_idx)) - rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) - rep.bits.global_xact_id := mem_idx + when(io.mem.resp.valid && (UFix(j) === init_client_id_arr(mem_idx))) { + rep.bits.payload.x_type := co.getTransactionReplyType(x_type_arr(mem_idx), sh_count_arr(mem_idx)) + rep.bits.payload.tile_xact_id := tile_xact_id_arr(mem_idx) + rep.bits.payload.global_xact_id := mem_idx rep.valid := Bool(true) } . otherwise { - rep.bits.x_type := co.getTransactionReplyType(x_type_arr(ack_idx), sh_count_arr(ack_idx)) - rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) - rep.bits.global_xact_id := ack_idx - when (UFix(j) === init_tile_id_arr(ack_idx)) { + rep.bits.payload.x_type := co.getTransactionReplyType(x_type_arr(ack_idx), sh_count_arr(ack_idx)) + rep.bits.payload.tile_xact_id := tile_xact_id_arr(ack_idx) + rep.bits.payload.global_xact_id := ack_idx + when (UFix(j) === init_client_id_arr(ack_idx)) { rep.valid := send_x_rep_ack_arr.toBits.orR sent_x_rep_ack_arr(ack_idx) := rep.ready } } } - io.mem.resp.ready := io.tiles(init_tile_id_arr(mem_idx)).xact_rep.ready + io.mem.resp.ready := io.tiles(init_client_id_arr(mem_idx)).xact_rep.ready // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests @@ -403,29 +404,29 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co for( j <- 0 until conf.ln.nTiles ) { val p_rep = io.tiles(j).probe_rep val p_rep_data = io.tiles(j).probe_rep_data - val idx = p_rep.bits.global_xact_id + val idx = p_rep.bits.payload.global_xact_id val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) val do_pop = foldR(pop_p_reps)(_ || _) p_rep.ready := Bool(true) - p_rep_data_dep_list(j).io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits) - p_rep_data_dep_list(j).io.enq.bits.global_xact_id := p_rep.bits.global_xact_id + p_rep_data_dep_list(j).io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits.payload) + p_rep_data_dep_list(j).io.enq.bits.global_xact_id := p_rep.bits.payload.global_xact_id p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) - when (p_rep.valid && co.messageHasData(p_rep.bits)) { + when (p_rep.valid && co.messageHasData(p_rep.bits.payload)) { p_data_valid_arr(idx) := Bool(true) - p_data_tile_id_arr(idx) := UFix(j) + p_data_client_id_arr(idx) := UFix(j) } p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) } for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid - trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits + trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_client_id).probe_rep_data.valid + trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_client_id).probe_rep_data.bits.payload - trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) - trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) + trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_client_id, p_rep_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) + trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_client_id, p_rep_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) for( j <- 0 until conf.ln.nTiles) { val p_rep = io.tiles(j).probe_rep - p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) + p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.payload.global_xact_id === UFix(i)) } } @@ -441,16 +442,16 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val conflicts = Vec(NGLOBAL_XACTS) { Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.addr) + conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.payload.addr) } - x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id - want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits))) + x_abort.bits.payload.tile_xact_id := x_init.bits.payload.tile_xact_id + want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits.payload))) x_abort.valid := Bool(false) switch(abort_state_arr(j)) { is(s_idle) { when(want_to_abort_arr(j)) { - when(co.messageHasData(x_init.bits)) { + when(co.messageHasData(x_init.bits.payload)) { abort_state_arr(j) := s_abort_drain } . otherwise { abort_state_arr(j) := s_abort_send @@ -485,10 +486,10 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co trackerList(i).io.alloc_req.bits := init_arb.io.out.bits trackerList(i).io.alloc_req.valid := init_arb.io.out.valid - trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits - trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid - trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) - trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) + trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_client_id).xact_init_data.bits.payload + trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_client_id).xact_init_data.valid + trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_client_id, x_init_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) + trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_client_id, x_init_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) } for( j <- 0 until conf.ln.nTiles ) { val x_init = io.tiles(j).xact_init @@ -496,11 +497,11 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val x_init_data_dep = x_init_data_dep_list(j).io.deq val x_abort = io.tiles(j).xact_abort init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid - init_arb.io.in(j).bits.xact_init := x_init.bits - init_arb.io.in(j).bits.tile_id := UFix(j) + init_arb.io.in(j).bits.xact_init := x_init.bits.payload + init_arb.io.in(j).bits.client_id := UFix(j) val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) val do_pop = foldR(pop_x_inits)(_||_) - x_init_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(x_init.bits) && (abort_state_arr(j) === s_idle) + x_init_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(x_init.bits.payload) && (abort_state_arr(j) === s_idle) x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) x_init.ready := (x_abort.valid && x_abort.ready) || do_pop x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) @@ -519,7 +520,7 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co p_req_arb_arr(j).io.in(i).valid := t.probe_req.valid && t.push_p_req(j) p_req_cnt_inc_arr(i)(j) := p_req_arb_arr(j).io.in(i).ready } - p_req_arb_arr(j).io.out <> io.tiles(j).probe_req + FIFOedLogicalNetworkIOWrapper(p_req_arb_arr(j).io.out) <> io.tiles(j).probe_req } } From fb2644760ff155c4b4dc14e954d65b5d40157c40 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 16 Jan 2013 23:57:35 -0800 Subject: [PATCH 155/374] single-ported coherence master --- uncore/src/package.scala | 6 +- uncore/src/uncore.scala | 344 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 347 insertions(+), 3 deletions(-) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 3c2a93df..2e73a6d7 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -112,11 +112,11 @@ class MasterSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { val xact_init = (new ClientSourcedIO){(new LogicalNetworkIO){new TransactionInit }} val xact_init_data = (new ClientSourcedIO){(new LogicalNetworkIO){new TransactionInitData }} - val xact_abort = (new MasterSourcedIO) {(new LogicalNetworkIO){new TransactionAbort }} - val probe_req = (new MasterSourcedIO) {(new LogicalNetworkIO){new ProbeRequest }} + val xact_abort = (new MasterSourcedIO){(new LogicalNetworkIO){new TransactionAbort }} + val probe_req = (new MasterSourcedIO){(new LogicalNetworkIO){new ProbeRequest }} val probe_rep = (new ClientSourcedIO){(new LogicalNetworkIO){new ProbeReply }} val probe_rep_data = (new ClientSourcedIO){(new LogicalNetworkIO){new ProbeReplyData }} - val xact_rep = (new MasterSourcedIO) {(new LogicalNetworkIO){new TransactionReply }} + val xact_rep = (new MasterSourcedIO){(new LogicalNetworkIO){new TransactionReply }} val xact_finish = (new ClientSourcedIO){(new LogicalNetworkIO){new TransactionFinish }} override def clone = { new TileLinkIO().asInstanceOf[this.type] } } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 697b632c..179179e8 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -524,3 +524,347 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co } } + +abstract class CoherenceAgent(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { + val io = new Bundle { + val network = (new TileLinkIO).flip + val incoherent = Vec(conf.nTiles) { Bool() }.asInput + val mem = new ioMem + } +} + +class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends CoherenceAgent()(conf.ln) +{ + implicit val lnConf = conf.ln + val co = conf.co + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) + val p_rep_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth must >= NPRIMARY + val x_init_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth should >= NPRIMARY + + for( i <- 0 until NGLOBAL_XACTS ) { + val t = trackerList(i) + t.io.tile_incoherent := io.incoherent.toBits + t.io.mem_resp.valid := io.mem.resp.valid && (io.mem.resp.bits.tag === UFix(i)) + t.io.mem_resp.bits := io.mem.resp.bits + } + io.mem.resp.ready := trackerList.map(_.io.mem_resp.ready).reduce(_||_) + + // Handle transaction initiation requests + // Only one allocation per cycle + // Init requests may or may not have data + val x_init = io.network.xact_init + val x_init_data = io.network.xact_init_data + val x_abort = io.network.xact_abort + val x_dep_deq = x_init_data_dep_q.io.deq + val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } + val abort_state = Reg(resetVal = s_idle) + val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) + val any_conflict = trackerList.map(_.io.has_conflict).reduce(_||_) + val all_busy = trackerList.map(_.io.busy).reduce(_&&_) + val want_to_abort = x_init.valid && (any_conflict || all_busy || (!x_init_data_dep_q.io.enq.ready && co.messageHasData(x_init.bits.payload))) + + val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } + for( i <- 0 until NGLOBAL_XACTS ) { + alloc_arb.io.in(i).valid := !trackerList(i).io.busy + trackerList(i).io.x_init.bits := x_init.bits + trackerList(i).io.x_init.valid := (abort_state === s_idle) && !want_to_abort && x_init.valid && alloc_arb.io.in(i).ready + + trackerList(i).io.x_init_data.bits := x_init_data.bits + trackerList(i).io.x_init_data.valid := x_init_data.valid + trackerList(i).io.x_init_data_dep.bits := x_dep_deq.bits + trackerList(i).io.x_init_data_dep.valid := x_dep_deq.valid + } + val pop_x_init = trackerList.map(_.io.x_init.ready).reduce(_||_) + x_init.ready := (x_abort.valid && x_abort.ready) || pop_x_init + x_init_data.ready := (abort_state === s_abort_drain) || trackerList.map(_.io.x_init_data.ready).reduce(_||_) + x_init_data_dep_q.io.enq.valid := pop_x_init && co.messageHasData(x_init.bits.payload) && (abort_state === s_idle) + x_init_data_dep_q.io.enq.bits.global_xact_id := OHToUFix(trackerList.map(_.io.x_init.ready)) + x_dep_deq.ready := trackerList.map(_.io.x_init_data_dep.ready).reduce(_||_) + + alloc_arb.io.out.ready := x_init.valid + + // Nack conflicting transaction init attempts + x_abort.bits.header.dst := x_init.bits.header.src + x_abort.bits.payload.tile_xact_id := x_init.bits.payload.tile_xact_id + x_abort.valid := Bool(false) + switch(abort_state) { + is(s_idle) { + when(want_to_abort) { + abort_state := Mux( co.messageHasData(x_init.bits.payload), s_abort_drain, s_abort_send) + } + } + is(s_abort_drain) { // raises x_init_data.ready below + when(x_init_data.valid) { + abort_cnt := abort_cnt + UFix(1) + when(abort_cnt === ~UFix(0, width = log2Up(REFILL_CYCLES))) { + abort_state := s_abort_send + } + } + } + is(s_abort_send) { // nothing is dequeued for now + x_abort.valid := Bool(true) + when(x_abort.ready) { // raises x_init.ready + abort_state := s_idle + } + } + } + + // Handle probe request generation + val p_req_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new ProbeRequest }} + for( i <- 0 until NGLOBAL_XACTS ) { + val t = trackerList(i).io + p_req_arb.io.in(i).bits := t.p_req.bits + p_req_arb.io.in(i).valid := t.p_req.valid + t.p_req.ready := p_req_arb.io.in(i).ready + } + io.network.probe_req <> p_req_arb.io.out + + // Handle probe replies, which may or may not have data + val p_rep = io.network.probe_rep + val p_rep_data = io.network.probe_rep_data + val idx = p_rep.bits.payload.global_xact_id + p_rep.ready := trackerList.map(_.io.p_rep.ready).reduce(_||_) + p_rep_data.ready := trackerList.map(_.io.p_rep_data.ready).reduce(_||_) + p_rep_data_dep_q.io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits.payload) + p_rep_data_dep_q.io.enq.bits.global_xact_id := p_rep.bits.payload.global_xact_id + p_rep_data_dep_q.io.deq.ready := trackerList.map(_.io.p_rep_data_dep.ready).reduce(_||_) + for( i <- 0 until NGLOBAL_XACTS ) { + trackerList(i).io.p_rep_data.valid := p_rep_data.valid + trackerList(i).io.p_rep_data.bits := p_rep_data.bits + trackerList(i).io.p_rep_data_dep.valid := p_rep_data_dep_q.io.deq.valid + trackerList(i).io.p_rep_data_dep.bits := p_rep_data_dep_q.io.deq.bits + trackerList(i).io.p_rep.valid := p_rep.valid && (idx === UFix(i)) + trackerList(i).io.p_rep.bits := p_rep.bits + } + + // Reply to initial requestor + // Forward memory responses from mem to tile or arbitrate to ack + val x_rep_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new TransactionReply }} + for( i <- 0 until NGLOBAL_XACTS ) { + val t = trackerList(i).io + x_rep_arb.io.in(i).bits := t.x_rep.bits + x_rep_arb.io.in(i).valid := t.x_rep.valid + t.x_rep.ready := x_rep_arb.io.in(i).ready + } + x_rep_arb.io.out.ready := Bool(false) + io.network.xact_rep.valid := x_rep_arb.io.out.valid + io.network.xact_rep.bits := x_rep_arb.io.out.bits + x_rep_arb.io.out.ready := io.network.xact_rep.ready + when(io.mem.resp.valid) { + io.network.xact_rep.valid := Bool(true) + io.network.xact_rep.bits := Vec(trackerList.map(_.io.x_rep.bits)){(new LogicalNetworkIO){new TransactionReply}}(io.mem.resp.bits.tag) + for( i <- 0 until NGLOBAL_XACTS ) { + trackerList(i).io.x_rep.ready := (io.mem.resp.bits.tag === UFix(i)) && io.network.xact_rep.ready + } + } + + // Free finished transactions + val finish = io.network.xact_finish + for( i <- 0 until NGLOBAL_XACTS ) { + trackerList(i).io.free := finish.valid && (finish.bits.payload.global_xact_id === UFix(i)) + } + finish.ready := Bool(true) + + // Create an arbiter for the one memory port + // We have to arbitrate between the different trackers' memory requests + // and once we have picked a request, get the right write data + val mem_req_cmd_arb = (new Arbiter(NGLOBAL_XACTS)) { new MemReqCmd() } + val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemData() } + for( i <- 0 until NGLOBAL_XACTS ) { + mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd + mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data + mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock + } + io.mem.req_cmd <> Queue(mem_req_cmd_arb.io.out) + io.mem.req_data <> Queue(mem_req_data_arb.io.out) +} + +class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Component { + val co = conf.co + implicit val ln = conf.ln + val io = new Bundle { + val x_init = (new FIFOIO){(new LogicalNetworkIO) { new TransactionInit }}.flip + val x_init_data = (new FIFOIO){(new LogicalNetworkIO) { new TransactionInitData }}.flip + val p_rep = (new FIFOIO){(new LogicalNetworkIO) { new ProbeReply }}.flip + val p_rep_data = (new FIFOIO){(new LogicalNetworkIO) { new ProbeReplyData }}.flip + val free = Bool(INPUT) + val tile_incoherent = Bits(INPUT, conf.ln.nTiles) + val p_rep_data_dep = (new FIFOIO) { new TrackerDependency }.flip + val x_init_data_dep = (new FIFOIO) { new TrackerDependency }.flip + val mem_resp = (new FIFOIO) { new MemResp }.flip + + val mem_req_cmd = (new FIFOIO) { new MemReqCmd } + val mem_req_data = (new FIFOIO) { new MemData } + val mem_req_lock = Bool(OUTPUT) + val p_req = (new FIFOIO) {(new LogicalNetworkIO) { new ProbeRequest }} + val x_rep = (new FIFOIO) {(new LogicalNetworkIO) { new TransactionReply }} + val busy = Bool(OUTPUT) + val has_conflict = Bool(OUTPUT) + } + + val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } + val state = Reg(resetVal = s_idle) + val xact = Reg{ new TransactionInit } + val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + //TODO: Will need id reg for merged release xacts + val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val p_rep_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val p_req_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) + val x_needs_read = Reg(resetVal = Bool(false)) + val x_init_data_needs_write = Reg(resetVal = Bool(false)) + val p_rep_data_needs_write = Reg(resetVal = Bool(false)) + val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) + val mem_cnt_next = mem_cnt + UFix(1) + val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) + val p_req_initial_flags = Bits(width = conf.ln.nTiles) + p_req_initial_flags := Bits(0) + if (conf.ln.nTiles > 1) { + // issue self-probes for uncached read xacts to facilitate I$ coherence + // TODO: this is hackish; figure out how to do it more systematically + val probe_self = co match { + case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.x_init.bits.payload) + case _ => Bool(false) + } + val myflag = Mux(probe_self, Bits(0), UFixToOH(io.x_init.bits.header.src(log2Up(conf.ln.nTiles)-1,0))) + p_req_initial_flags := ~(io.tile_incoherent | myflag) + } + val all_x_reps_require_acks = Bool(true) + + io.busy := state != s_idle + io.has_conflict := co.isCoherenceConflict(xact.addr, io.x_init.bits.payload.addr) && (state != s_idle) + io.mem_req_cmd.valid := Bool(false) + io.mem_req_cmd.bits.rw := Bool(false) + io.mem_req_cmd.bits.addr := xact.addr + io.mem_req_cmd.bits.tag := UFix(id) + io.mem_req_data.valid := Bool(false) + io.mem_req_data.bits.data := UFix(0) + io.mem_req_lock := Bool(false) + io.p_req.valid := Bool(false) + io.p_req.bits.payload.p_type := co.getProbeRequestType(xact.x_type, UFix(0)) + io.p_req.bits.payload.global_xact_id := UFix(id) + io.p_req.bits.payload.addr := xact.addr + io.p_req.bits.header.dst := UFix(0) + io.x_rep.bits.payload.data := io.mem_resp.bits.data + io.x_rep.bits.payload.x_type := co.getTransactionReplyType(xact.x_type, init_sharer_cnt_) + io.x_rep.bits.payload.tile_xact_id := xact.tile_xact_id + io.x_rep.bits.payload.global_xact_id := UFix(id) + io.x_rep.bits.payload.require_ack := all_x_reps_require_acks + io.x_rep.bits.header.dst := init_client_id_ + io.x_rep.valid := (io.mem_resp.valid && (UFix(id) === io.mem_resp.bits.tag)) + io.x_init.ready := Bool(false) + io.x_init_data.ready := Bool(false) + io.x_init_data_dep.ready := Bool(false) + io.p_rep.ready := Bool(false) + io.p_rep_data.ready := Bool(false) + io.p_rep_data_dep.ready := Bool(false) + io.mem_resp.ready := io.x_rep.ready + + switch (state) { + is(s_idle) { + when( io.x_init.valid ) { + xact := io.x_init.bits.payload + init_client_id_ := io.x_init.bits.header.src + init_sharer_cnt_ := UFix(conf.ln.nTiles) // TODO: Broadcast only + x_init_data_needs_write := co.messageHasData(io.x_init.bits.payload) + x_needs_read := co.needsMemRead(io.x_init.bits.payload.x_type, UFix(0)) + p_req_flags := p_req_initial_flags + mem_cnt := UFix(0) + p_w_mem_cmd_sent := Bool(false) + x_w_mem_cmd_sent := Bool(false) + io.x_init.ready := Bool(true) + if(conf.ln.nTiles > 1) { + p_rep_count := PopCount(p_req_initial_flags) + state := Mux(p_req_initial_flags.orR, s_probe, s_mem) + } else state := s_mem + } + } + is(s_probe) { + val curr_p_id = PriorityEncoder(p_req_flags) + when(p_req_flags.orR) { + io.p_req.valid := Bool(true) + io.p_req.bits.header.dst := curr_p_id + } + when(io.p_req.ready) { + p_req_flags := p_req_flags & ~(UFixToOH(curr_p_id)) + } + when(io.p_rep.valid) { + io.p_rep.ready := Bool(true) + if(conf.ln.nTiles > 1) p_rep_count := p_rep_count - UFix(1) + when(p_rep_count === UFix(1)) { + state := s_mem + } + p_rep_data_needs_write := co.messageHasData(io.p_rep.bits.payload) + } + } + is(s_mem) { + when (p_rep_data_needs_write) { + doMemReqWrite(io.mem_req_cmd, + io.mem_req_data, + io.mem_req_lock, + io.p_rep_data, + p_rep_data_needs_write, + p_w_mem_cmd_sent, + io.p_rep_data_dep.ready, + io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id))) + } . elsewhen(x_init_data_needs_write) { + doMemReqWrite(io.mem_req_cmd, + io.mem_req_data, + io.mem_req_lock, + io.x_init_data, + x_init_data_needs_write, + x_w_mem_cmd_sent, + io.x_init_data_dep.ready, + io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id))) + } . elsewhen (x_needs_read) { + doMemReqRead(io.mem_req_cmd, x_needs_read) + } . otherwise { + state := Mux(co.needsAckReply(xact.x_type, UFix(0)), s_ack, + Mux(all_x_reps_require_acks, s_busy, s_idle)) + } + } + is(s_ack) { + io.x_rep.valid := Bool(true) + when(io.x_rep.ready) { state := Mux(all_x_reps_require_acks, s_busy, s_idle) } + } + is(s_busy) { // Nothing left to do but wait for transaction to complete + when (io.free) { + state := s_idle + } + } + } + + def doMemReqWrite[T <: Data](req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: FIFOIO[LogicalNetworkIO[T]], trigger: Bool, cmd_sent: Bool, pop_dep: Bool, at_front_of_dep_queue: Bool) { + req_cmd.bits.rw := Bool(true) + req_data.bits := data.bits.payload + when(req_cmd.ready && req_cmd.valid) { + cmd_sent := Bool(true) + } + when (at_front_of_dep_queue) { + req_cmd.valid := !cmd_sent && req_data.ready && data.valid + lock := data.valid || cmd_sent + when (req_cmd.ready || cmd_sent) { + req_data.valid := data.valid + when(req_data.ready) { + data.ready:= Bool(true) + when (data.valid) { + mem_cnt := mem_cnt_next + when(mem_cnt === UFix(REFILL_CYCLES-1)) { + pop_dep := Bool(true) + trigger := Bool(false) + } + } + } + } + } + } + + def doMemReqRead(req_cmd: FIFOIO[MemReqCmd], trigger: Bool) { + req_cmd.valid := Bool(true) + req_cmd.bits.rw := Bool(false) + when(req_cmd.ready) { + trigger := Bool(false) + } + } +} From c211d74e95ed1561cca1827650b93087b76bcc23 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 21 Jan 2013 17:17:26 -0800 Subject: [PATCH 156/374] New TileLink names --- uncore/src/coherence.scala | 898 ++++++++++++++++++------------------- uncore/src/consts.scala | 20 +- uncore/src/package.scala | 16 +- uncore/src/tilelink.scala | 114 ++--- uncore/src/uncore.scala | 286 ++++++------ 5 files changed, 668 insertions(+), 666 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index caa2b648..4268fcf8 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -15,7 +15,7 @@ abstract class CoherencePolicy { def isHit (cmd: Bits, state: UFix): Bool def isValid (state: UFix): Bool - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool def needsWriteback (state: UFix): Bool @@ -23,71 +23,71 @@ abstract class CoherencePolicy { def newStateOnCacheControl(cmd: Bits): UFix def newStateOnWriteback(): UFix def newStateOnFlush(): UFix - def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix - def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix + def newStateOnProbe(incoming: Probe, state: UFix): Bits - def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix - def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix - def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits - def getTransactionInitTypeOnWriteback(): Bits + def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix + def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix + def getAcquireTypeOnCacheControl(cmd: Bits): Bits + def getAcquireTypeOnWriteback(): Bits - def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply + def newRelease (incoming: Probe, state: UFix): Release - def messageHasData (reply: ProbeReply): Bool - def messageHasData (init: TransactionInit): Bool - def messageHasData (reply: TransactionReply): Bool - def messageUpdatesDataArray (reply: TransactionReply): Bool - def messageIsUncached(init: TransactionInit): Bool + def messageHasData (reply: Release): Bool + def messageHasData (acq: Acquire): Bool + def messageHasData (reply: Grant): Bool + def messageUpdatesDataArray (reply: Grant): Bool + def messageIsUncached(acq: Acquire): Bool def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool - def getTransactionReplyType(x_type: UFix, count: UFix): Bits - def getProbeRequestType(x_type: UFix, global_state: UFix): UFix - def needsMemRead(x_type: UFix, global_state: UFix): Bool - def needsMemWrite(x_type: UFix, global_state: UFix): Bool - def needsAckReply(x_type: UFix, global_state: UFix): Bool + def getGrantType(a_type: UFix, count: UFix): Bits + def getProbeType(a_type: UFix, global_state: UFix): UFix + def needsMemRead(a_type: UFix, global_state: UFix): Bool + def needsMemWrite(a_type: UFix, global_state: UFix): Bool + def needsAckReply(a_type: UFix, global_state: UFix): Bool } trait UncachedTransactions { - def getUncachedReadTransactionInit(addr: UFix, id: UFix): TransactionInit - def getUncachedWriteTransactionInit(addr: UFix, id: UFix): TransactionInit - def getUncachedReadWordTransactionInit(addr: UFix, id: UFix): TransactionInit - def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits): TransactionInit - def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix): TransactionInit - def isUncachedReadTransaction(xinit: TransactionInit): Bool + def getUncachedReadAcquire(addr: UFix, id: UFix): Acquire + def getUncachedWriteAcquire(addr: UFix, id: UFix): Acquire + def getUncachedReadWordAcquire(addr: UFix, id: UFix): Acquire + def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits): Acquire + def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix): Acquire + def isUncachedReadTransaction(acq: Acquire): Bool } abstract class CoherencePolicyWithUncached extends CoherencePolicy with UncachedTransactions abstract class IncoherentPolicy extends CoherencePolicy { // UNIMPLEMENTED - def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = state - def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - val reply = new ProbeReply() - reply.p_type := UFix(0) - reply.global_xact_id := UFix(0) + def newStateOnProbe(incoming: Probe, state: UFix): Bits = state + def newRelease (incoming: Probe, state: UFix): Release = { + val reply = new Release + reply.r_type := UFix(0) + reply.master_xact_id := UFix(0) reply } - def messageHasData (reply: ProbeReply) = Bool(false) + def messageHasData (reply: Release) = Bool(false) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = Bool(false) - def getTransactionReplyType(x_type: UFix, count: UFix): Bits = Bits(0) - def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = UFix(0) - def needsMemRead(x_type: UFix, global_state: UFix): Bool = Bool(false) - def needsMemWrite(x_type: UFix, global_state: UFix): Bool = Bool(false) - def needsAckReply(x_type: UFix, global_state: UFix): Bool = Bool(false) + def getGrantType(a_type: UFix, count: UFix): Bits = Bits(0) + def getProbeType(a_type: UFix, global_state: UFix): UFix = UFix(0) + def needsMemRead(a_type: UFix, global_state: UFix): Bool = Bool(false) + def needsMemWrite(a_type: UFix, global_state: UFix): Bool = Bool(false) + def needsAckReply(a_type: UFix, global_state: UFix): Bool = Bool(false) } class ThreeStateIncoherence extends IncoherentPolicy { val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } - val xactInitReadClean :: xactInitReadDirty :: xactInitWriteback :: Nil = Enum(3){ UFix() } - val xactReplyData :: xactReplyAck :: Nil = Enum(2){ UFix() } - val probeRepInvalidateAck :: Nil = Enum(1){ UFix() } + val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(3){ UFix() } + val grantData :: grantAck :: Nil = Enum(2){ UFix() } + val releaseInvalidateAck :: Nil = Enum(1){ UFix() } val uncachedTypeList = List() - val hasDataTypeList = List(xactInitWriteback) + val hasDataTypeList = List(acquireWriteback) def isHit ( cmd: Bits, state: UFix): Bool = (state === tileClean || state === tileDirty) def isValid (state: UFix): Bool = state != tileInvalid - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit) = Bool(false) + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire) = Bool(false) def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = state === tileDirty def needsWriteback (state: UFix): Bool = state === tileDirty @@ -99,28 +99,28 @@ class ThreeStateIncoherence extends IncoherentPolicy { def newStateOnCacheControl(cmd: Bits) = tileInvalid //TODO def newStateOnWriteback() = tileInvalid def newStateOnFlush() = tileInvalid - def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit) = { - MuxLookup(incoming.x_type, tileInvalid, Array( - xactReplyData -> Mux(outstanding.x_type === xactInitReadDirty, tileDirty, tileClean), - xactReplyAck -> tileInvalid + def newStateOnGrant(incoming: Grant, outstanding: Acquire) = { + MuxLookup(incoming.g_type, tileInvalid, Array( + grantData -> Mux(outstanding.a_type === acquireReadDirty, tileDirty, tileClean), + grantAck -> tileInvalid )) } - def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, xactInitReadDirty, xactInitReadClean) + Mux(write || cmd === M_PFW, acquireReadDirty, acquireReadClean) } - def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, xactInitReadDirty, outstanding.x_type) + Mux(write, acquireReadDirty, outstanding.a_type) } - def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteback //TODO - def getTransactionInitTypeOnWriteback(): Bits = xactInitWriteback + def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteback //TODO + def getAcquireTypeOnWriteback(): Bits = acquireWriteback - def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_) - def messageHasData (reply: TransactionReply) = (reply.x_type === xactReplyData) - def messageUpdatesDataArray (reply: TransactionReply) = (reply.x_type === xactReplyData) - def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_) + def messageHasData (init: Acquire): Bool = hasDataTypeList.map(t => init.a_type === t).reduceLeft(_||_) + def messageHasData (reply: Grant) = (reply.g_type === grantData) + def messageUpdatesDataArray (reply: Grant) = (reply.g_type === grantData) + def messageIsUncached(init: Acquire): Bool = uncachedTypeList.map(t => init.a_type === t).reduceLeft(_||_) } class MICoherence extends CoherencePolicyWithUncached { @@ -128,17 +128,17 @@ class MICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileValid :: Nil = Enum(2){ UFix() } val globalInvalid :: globalValid :: Nil = Enum(2){ UFix() } - val xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: Nil = Enum(6){ UFix() } - val xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: Nil = Enum(6){ UFix() } - val probeReqInvalidate :: probeReqCopy :: Nil = Enum(2){ UFix() } - val probeRepInvalidateData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepCopyAck :: Nil = Enum(4){ UFix() } - val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactReplyReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) - val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(6){ UFix() } + val grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(6){ UFix() } + val probeInvalidate :: probeCopy :: Nil = Enum(2){ UFix() } + val releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(4){ UFix() } + val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, grantReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = state != tileInvalid def isValid (state: UFix): Bool = state != tileInvalid - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = (outstanding.x_type != xactInitReadExclusive) + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = (outstanding.a_type != acquireReadExclusive) def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { MuxLookup(cmd, (state === tileValid), Array( M_INV -> (state === tileValid), @@ -158,95 +158,95 @@ class MICoherence extends CoherencePolicyWithUncached { } def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { - MuxLookup(incoming.x_type, tileInvalid, Array( - xactReplyReadExclusive -> tileValid, - xactReplyReadUncached -> tileInvalid, - xactReplyWriteUncached -> tileInvalid, - xactReplyReadWordUncached -> tileInvalid, - xactReplyWriteWordUncached -> tileInvalid, - xactReplyAtomicUncached -> tileInvalid + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix = { + MuxLookup(incoming.g_type, tileInvalid, Array( + grantReadExclusive -> tileValid, + grantReadUncached -> tileInvalid, + grantWriteUncached -> tileInvalid, + grantReadWordUncached -> tileInvalid, + grantWriteWordUncached -> tileInvalid, + grantAtomicUncached -> tileInvalid )) } - def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { + def newStateOnProbe(incoming: Probe, state: UFix): Bits = { MuxLookup(incoming.p_type, state, Array( - probeReqInvalidate -> tileInvalid, - probeReqCopy -> state + probeInvalidate -> tileInvalid, + probeCopy -> state )) } - def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id) - def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id) - def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) - def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) - def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) - def isUncachedReadTransaction(xinit: TransactionInit) = xinit.x_type === xactInitReadUncached + def getUncachedReadAcquire(addr: UFix, id: UFix) = Acquire(acquireReadUncached, addr, id) + def getUncachedWriteAcquire(addr: UFix, id: UFix) = Acquire(acquireWriteUncached, addr, id) + def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) + def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) + def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) + def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached - def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = xactInitReadExclusive - def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = xactInitReadExclusive - def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached - def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) + def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = acquireReadExclusive + def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = acquireReadExclusive + def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteUncached + def getAcquireTypeOnWriteback(): Bits = getAcquireTypeOnCacheControl(M_INV) - def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - val reply = new ProbeReply() - val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( - probeReqInvalidate -> probeRepInvalidateData, - probeReqCopy -> probeRepCopyData + def newRelease (incoming: Probe, state: UFix): Release = { + val reply = new Release + val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( + probeInvalidate -> releaseInvalidateData, + probeCopy -> releaseCopyData )) - val without_data = MuxLookup(incoming.p_type, probeRepInvalidateAck, Array( - probeReqInvalidate -> probeRepInvalidateAck, - probeReqCopy -> probeRepCopyAck + val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> releaseInvalidateAck, + probeCopy -> releaseCopyAck )) - reply.p_type := Mux(needsWriteback(state), with_data, without_data) - reply.global_xact_id := incoming.global_xact_id + reply.r_type := Mux(needsWriteback(state), with_data, without_data) + reply.master_xact_id := incoming.master_xact_id reply } - def messageHasData (reply: ProbeReply): Bool = { - (reply.p_type === probeRepInvalidateData || - reply.p_type === probeRepCopyData) + def messageHasData (reply: Release): Bool = { + (reply.r_type === releaseInvalidateData || + reply.r_type === releaseCopyData) } - def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_) - def messageHasData (reply: TransactionReply): Bool = { - (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyWriteWordUncached) + def messageHasData (acq: Acquire): Bool = hasDataTypeList.map(t => acq.a_type === t).reduceLeft(_||_) + def messageHasData (reply: Grant): Bool = { + (reply.g_type != grantWriteUncached && reply.g_type != grantWriteWordUncached) } - def messageUpdatesDataArray (reply: TransactionReply): Bool = { - (reply.x_type === xactReplyReadExclusive) + def messageUpdatesDataArray (reply: Grant): Bool = { + (reply.g_type === grantReadExclusive) } - def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_) + def messageIsUncached(acq: Acquire): Bool = uncachedTypeList.map(t => acq.a_type === t).reduceLeft(_||_) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getTransactionReplyType(x_type: UFix, count: UFix): Bits = { - MuxLookup(x_type, xactReplyReadUncached, Array( - xactInitReadExclusive -> xactReplyReadExclusive, - xactInitReadUncached -> xactReplyReadUncached, - xactInitWriteUncached -> xactReplyWriteUncached, - xactInitReadWordUncached -> xactReplyReadWordUncached, - xactInitWriteWordUncached -> xactReplyWriteWordUncached, - xactInitAtomicUncached -> xactReplyAtomicUncached + def getGrantType(a_type: UFix, count: UFix): Bits = { + MuxLookup(a_type, grantReadUncached, Array( + acquireReadExclusive -> grantReadExclusive, + acquireReadUncached -> grantReadUncached, + acquireWriteUncached -> grantWriteUncached, + acquireReadWordUncached -> grantReadWordUncached, + acquireWriteWordUncached -> grantWriteWordUncached, + acquireAtomicUncached -> grantAtomicUncached )) } - def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = { - MuxLookup(x_type, probeReqCopy, Array( - xactInitReadExclusive -> probeReqInvalidate, - xactInitReadUncached -> probeReqCopy, - xactInitWriteUncached -> probeReqInvalidate, - xactInitReadWordUncached -> probeReqCopy, - xactInitWriteWordUncached -> probeReqInvalidate, - xactInitAtomicUncached -> probeReqInvalidate + def getProbeType(a_type: UFix, global_state: UFix): UFix = { + MuxLookup(a_type, probeCopy, Array( + acquireReadExclusive -> probeInvalidate, + acquireReadUncached -> probeCopy, + acquireWriteUncached -> probeInvalidate, + acquireReadWordUncached -> probeCopy, + acquireWriteWordUncached -> probeInvalidate, + acquireAtomicUncached -> probeInvalidate )) } - def needsMemRead(x_type: UFix, global_state: UFix): Bool = { - (x_type != xactInitWriteUncached) + def needsMemRead(a_type: UFix, global_state: UFix): Bool = { + (a_type != acquireWriteUncached) } - def needsMemWrite(x_type: UFix, global_state: UFix): Bool = { - (x_type === xactInitWriteUncached) + def needsMemWrite(a_type: UFix, global_state: UFix): Bool = { + (a_type === acquireWriteUncached) } - def needsAckReply(x_type: UFix, global_state: UFix): Bool = { - (x_type === xactInitWriteUncached) + def needsAckReply(a_type: UFix, global_state: UFix): Bool = { + (a_type === acquireWriteUncached) } } @@ -255,20 +255,20 @@ class MEICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(3){ UFix() } val globalInvalid :: globalExclusiveClean :: Nil = Enum(2){ UFix() } - val xactInitReadExclusiveClean :: xactInitReadExclusiveDirty :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: Nil = Enum(7){ UFix() } - val xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: Nil = Enum(7){ UFix() } - val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() } - val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() } - val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactReplyReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) - val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(7){ UFix() } + val grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(7){ UFix() } + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(6){ UFix() } + val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, grantReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = state != tileInvalid def isValid (state: UFix): Bool = state != tileInvalid - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { val (read, write) = cpuCmdToRW(cmd) (read && messageIsUncached(outstanding)) || - (write && (outstanding.x_type != xactInitReadExclusiveDirty)) + (write && (outstanding.a_type != acquireReadExclusiveDirty)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( @@ -292,108 +292,108 @@ class MEICoherence extends CoherencePolicyWithUncached { } def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { - MuxLookup(incoming.x_type, tileInvalid, Array( - xactReplyReadExclusive -> Mux(outstanding.x_type === xactInitReadExclusiveDirty, tileExclusiveDirty, tileExclusiveClean), - xactReplyReadExclusiveAck -> tileExclusiveDirty, - xactReplyReadUncached -> tileInvalid, - xactReplyWriteUncached -> tileInvalid, - xactReplyReadWordUncached -> tileInvalid, - xactReplyWriteWordUncached -> tileInvalid, - xactReplyAtomicUncached -> tileInvalid + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix = { + MuxLookup(incoming.g_type, tileInvalid, Array( + grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusiveDirty, tileExclusiveDirty, tileExclusiveClean), + grantReadExclusiveAck -> tileExclusiveDirty, + grantReadUncached -> tileInvalid, + grantWriteUncached -> tileInvalid, + grantReadWordUncached -> tileInvalid, + grantWriteWordUncached -> tileInvalid, + grantAtomicUncached -> tileInvalid )) } - def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { + def newStateOnProbe(incoming: Probe, state: UFix): Bits = { MuxLookup(incoming.p_type, state, Array( - probeReqInvalidate -> tileInvalid, - probeReqDowngrade -> tileExclusiveClean, - probeReqCopy -> state + probeInvalidate -> tileInvalid, + probeDowngrade -> tileExclusiveClean, + probeCopy -> state )) } - def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id) - def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id) - def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) - def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) - def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) - def isUncachedReadTransaction(xinit: TransactionInit) = xinit.x_type === xactInitReadUncached + def getUncachedReadAcquire(addr: UFix, id: UFix) = Acquire(acquireReadUncached, addr, id) + def getUncachedWriteAcquire(addr: UFix, id: UFix) = Acquire(acquireWriteUncached, addr, id) + def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) + def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) + def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) + def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached - def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, xactInitReadExclusiveDirty, xactInitReadExclusiveClean) + Mux(write, acquireReadExclusiveDirty, acquireReadExclusiveClean) } - def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, xactInitReadExclusiveDirty, outstanding.x_type) + Mux(write, acquireReadExclusiveDirty, outstanding.a_type) } - def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached - def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) + def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteUncached + def getAcquireTypeOnWriteback(): Bits = getAcquireTypeOnCacheControl(M_INV) - def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - val reply = new ProbeReply() - val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( - probeReqInvalidate -> probeRepInvalidateData, - probeReqDowngrade -> probeRepDowngradeData, - probeReqCopy -> probeRepCopyData + def newRelease (incoming: Probe, state: UFix): Release = { + val reply = new Release + val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( + probeInvalidate -> releaseInvalidateData, + probeDowngrade -> releaseDowngradeData, + probeCopy -> releaseCopyData )) - val without_data = MuxLookup(incoming.p_type, probeRepInvalidateAck, Array( - probeReqInvalidate -> probeRepInvalidateAck, - probeReqDowngrade -> probeRepDowngradeAck, - probeReqCopy -> probeRepCopyAck + val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> releaseInvalidateAck, + probeDowngrade -> releaseDowngradeAck, + probeCopy -> releaseCopyAck )) - reply.p_type := Mux(needsWriteback(state), with_data, without_data) - reply.global_xact_id := incoming.global_xact_id + reply.r_type := Mux(needsWriteback(state), with_data, without_data) + reply.master_xact_id := incoming.master_xact_id reply } - def messageHasData (reply: ProbeReply): Bool = { - (reply.p_type === probeRepInvalidateData || - reply.p_type === probeRepDowngradeData || - reply.p_type === probeRepCopyData) + def messageHasData (reply: Release): Bool = { + (reply.r_type === releaseInvalidateData || + reply.r_type === releaseDowngradeData || + reply.r_type === releaseCopyData) } - def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_) - def messageHasData (reply: TransactionReply): Bool = { - (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck && reply.x_type != xactReplyWriteWordUncached) + def messageHasData (acq: Acquire): Bool = hasDataTypeList.map(t => acq.a_type === t).reduceLeft(_||_) + def messageHasData (reply: Grant): Bool = { + (reply.g_type != grantWriteUncached && reply.g_type != grantReadExclusiveAck && reply.g_type != grantWriteWordUncached) } - def messageUpdatesDataArray (reply: TransactionReply): Bool = { - (reply.x_type === xactReplyReadExclusive) + def messageUpdatesDataArray (reply: Grant): Bool = { + (reply.g_type === grantReadExclusive) } - def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_) + def messageIsUncached(init: Acquire): Bool = uncachedTypeList.map(t => init.a_type === t).reduceLeft(_||_) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getTransactionReplyType(x_type: UFix, count: UFix): Bits = { - MuxLookup(x_type, xactReplyReadUncached, Array( - xactInitReadExclusiveClean -> xactReplyReadExclusive, - xactInitReadExclusiveDirty -> xactReplyReadExclusive, - xactInitReadUncached -> xactReplyReadUncached, - xactInitWriteUncached -> xactReplyWriteUncached, - xactInitReadWordUncached -> xactReplyReadWordUncached, - xactInitWriteWordUncached -> xactReplyWriteWordUncached, - xactInitAtomicUncached -> xactReplyAtomicUncached + def getGrantType(a_type: UFix, count: UFix): Bits = { + MuxLookup(a_type, grantReadUncached, Array( + acquireReadExclusiveClean -> grantReadExclusive, + acquireReadExclusiveDirty -> grantReadExclusive, + acquireReadUncached -> grantReadUncached, + acquireWriteUncached -> grantWriteUncached, + acquireReadWordUncached -> grantReadWordUncached, + acquireWriteWordUncached -> grantWriteWordUncached, + acquireAtomicUncached -> grantAtomicUncached )) } - def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = { - MuxLookup(x_type, probeReqCopy, Array( - xactInitReadExclusiveClean -> probeReqInvalidate, - xactInitReadExclusiveDirty -> probeReqInvalidate, - xactInitReadUncached -> probeReqCopy, - xactInitWriteUncached -> probeReqInvalidate, - xactInitReadWordUncached -> probeReqCopy, - xactInitWriteWordUncached -> probeReqInvalidate, - xactInitAtomicUncached -> probeReqInvalidate + def getProbeType(a_type: UFix, global_state: UFix): UFix = { + MuxLookup(a_type, probeCopy, Array( + acquireReadExclusiveClean -> probeInvalidate, + acquireReadExclusiveDirty -> probeInvalidate, + acquireReadUncached -> probeCopy, + acquireWriteUncached -> probeInvalidate, + acquireReadWordUncached -> probeCopy, + acquireWriteWordUncached -> probeInvalidate, + acquireAtomicUncached -> probeInvalidate )) } - def needsMemRead(x_type: UFix, global_state: UFix): Bool = { - (x_type != xactInitWriteUncached) + def needsMemRead(a_type: UFix, global_state: UFix): Bool = { + (a_type != acquireWriteUncached) } - def needsMemWrite(x_type: UFix, global_state: UFix): Bool = { - (x_type === xactInitWriteUncached) + def needsMemWrite(a_type: UFix, global_state: UFix): Bool = { + (a_type === acquireWriteUncached) } - def needsAckReply(x_type: UFix, global_state: UFix): Bool = { - (x_type === xactInitWriteUncached) + def needsAckReply(a_type: UFix, global_state: UFix): Bool = { + (a_type === acquireWriteUncached) } } @@ -402,12 +402,12 @@ class MSICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileShared :: tileExclusiveDirty :: Nil = Enum(3){ UFix() } val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(3){ UFix() } - val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: Nil = Enum(7){ UFix() } - val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: Nil = Enum(8){ UFix() } - val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() } - val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() } - val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactReplyReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) - val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(7){ UFix() } + val grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(8){ UFix() } + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(6){ UFix() } + val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, grantReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) @@ -418,10 +418,10 @@ class MSICoherence extends CoherencePolicyWithUncached { state != tileInvalid } - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { val (read, write) = cpuCmdToRW(cmd) (read && messageIsUncached(outstanding)) || - (write && (outstanding.x_type != xactInitReadExclusive)) + (write && (outstanding.a_type != acquireReadExclusive)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( @@ -445,106 +445,106 @@ class MSICoherence extends CoherencePolicyWithUncached { } def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { - MuxLookup(incoming.x_type, tileInvalid, Array( - xactReplyReadShared -> tileShared, - xactReplyReadExclusive -> tileExclusiveDirty, - xactReplyReadExclusiveAck -> tileExclusiveDirty, - xactReplyReadUncached -> tileInvalid, - xactReplyWriteUncached -> tileInvalid, - xactReplyReadWordUncached -> tileInvalid, - xactReplyWriteWordUncached -> tileInvalid, - xactReplyAtomicUncached -> tileInvalid + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix = { + MuxLookup(incoming.g_type, tileInvalid, Array( + grantReadShared -> tileShared, + grantReadExclusive -> tileExclusiveDirty, + grantReadExclusiveAck -> tileExclusiveDirty, + grantReadUncached -> tileInvalid, + grantWriteUncached -> tileInvalid, + grantReadWordUncached -> tileInvalid, + grantWriteWordUncached -> tileInvalid, + grantAtomicUncached -> tileInvalid )) } - def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { + def newStateOnProbe(incoming: Probe, state: UFix): Bits = { MuxLookup(incoming.p_type, state, Array( - probeReqInvalidate -> tileInvalid, - probeReqDowngrade -> tileShared, - probeReqCopy -> state + probeInvalidate -> tileInvalid, + probeDowngrade -> tileShared, + probeCopy -> state )) } - def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id) - def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id) - def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) - def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) - def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) - def isUncachedReadTransaction(xinit: TransactionInit) = xinit.x_type === xactInitReadUncached + def getUncachedReadAcquire(addr: UFix, id: UFix) = Acquire(acquireReadUncached, addr, id) + def getUncachedWriteAcquire(addr: UFix, id: UFix) = Acquire(acquireWriteUncached, addr, id) + def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) + def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) + def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) + def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached - def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, xactInitReadExclusive, xactInitReadShared) + Mux(write || cmd === M_PFW, acquireReadExclusive, acquireReadShared) } - def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, xactInitReadExclusive, outstanding.x_type) + Mux(write, acquireReadExclusive, outstanding.a_type) } - def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached - def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) + def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteUncached + def getAcquireTypeOnWriteback(): Bits = getAcquireTypeOnCacheControl(M_INV) - def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - val reply = new ProbeReply() - val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( - probeReqInvalidate -> probeRepInvalidateData, - probeReqDowngrade -> probeRepDowngradeData, - probeReqCopy -> probeRepCopyData + def newRelease (incoming: Probe, state: UFix): Release = { + val reply = new Release + val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( + probeInvalidate -> releaseInvalidateData, + probeDowngrade -> releaseDowngradeData, + probeCopy -> releaseCopyData )) - val without_data = MuxLookup(incoming.p_type, probeRepInvalidateAck, Array( - probeReqInvalidate -> probeRepInvalidateAck, - probeReqDowngrade -> probeRepDowngradeAck, - probeReqCopy -> probeRepCopyAck + val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> releaseInvalidateAck, + probeDowngrade -> releaseDowngradeAck, + probeCopy -> releaseCopyAck )) - reply.p_type := Mux(needsWriteback(state), with_data, without_data) - reply.global_xact_id := incoming.global_xact_id + reply.r_type := Mux(needsWriteback(state), with_data, without_data) + reply.master_xact_id := incoming.master_xact_id reply } - def messageHasData (reply: ProbeReply): Bool = { - (reply.p_type === probeRepInvalidateData || - reply.p_type === probeRepDowngradeData || - reply.p_type === probeRepCopyData) + def messageHasData (reply: Release): Bool = { + (reply.r_type === releaseInvalidateData || + reply.r_type === releaseDowngradeData || + reply.r_type === releaseCopyData) } - def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_) - def messageHasData (reply: TransactionReply): Bool = { - (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck && reply.x_type != xactReplyWriteWordUncached) + def messageHasData (acq: Acquire): Bool = hasDataTypeList.map(t => acq.a_type === t).reduceLeft(_||_) + def messageHasData (reply: Grant): Bool = { + (reply.g_type != grantWriteUncached && reply.g_type != grantReadExclusiveAck && reply.g_type != grantWriteWordUncached) } - def messageUpdatesDataArray (reply: TransactionReply): Bool = { - (reply.x_type === xactReplyReadShared || reply.x_type === xactReplyReadExclusive) + def messageUpdatesDataArray (reply: Grant): Bool = { + (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } - def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_) + def messageIsUncached(acq: Acquire): Bool = uncachedTypeList.map(t => acq.a_type === t).reduceLeft(_||_) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getTransactionReplyType(x_type: UFix, count: UFix): Bits = { - MuxLookup(x_type, xactReplyReadUncached, Array( - xactInitReadShared -> Mux(count > UFix(0), xactReplyReadShared, xactReplyReadExclusive), - xactInitReadExclusive -> xactReplyReadExclusive, - xactInitReadUncached -> xactReplyReadUncached, - xactInitWriteUncached -> xactReplyWriteUncached, - xactInitReadWordUncached -> xactReplyReadWordUncached, - xactInitWriteWordUncached -> xactReplyWriteWordUncached, - xactInitAtomicUncached -> xactReplyAtomicUncached + def getGrantType(a_type: UFix, count: UFix): Bits = { + MuxLookup(a_type, grantReadUncached, Array( + acquireReadShared -> Mux(count > UFix(0), grantReadShared, grantReadExclusive), + acquireReadExclusive -> grantReadExclusive, + acquireReadUncached -> grantReadUncached, + acquireWriteUncached -> grantWriteUncached, + acquireReadWordUncached -> grantReadWordUncached, + acquireWriteWordUncached -> grantWriteWordUncached, + acquireAtomicUncached -> grantAtomicUncached )) } - def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = { - MuxLookup(x_type, probeReqCopy, Array( - xactInitReadShared -> probeReqDowngrade, - xactInitReadExclusive -> probeReqInvalidate, - xactInitReadUncached -> probeReqCopy, - xactInitWriteUncached -> probeReqInvalidate + def getProbeType(a_type: UFix, global_state: UFix): UFix = { + MuxLookup(a_type, probeCopy, Array( + acquireReadShared -> probeDowngrade, + acquireReadExclusive -> probeInvalidate, + acquireReadUncached -> probeCopy, + acquireWriteUncached -> probeInvalidate )) } - def needsMemRead(x_type: UFix, global_state: UFix): Bool = { - (x_type != xactInitWriteUncached) + def needsMemRead(a_type: UFix, global_state: UFix): Bool = { + (a_type != acquireWriteUncached) } - def needsMemWrite(x_type: UFix, global_state: UFix): Bool = { - (x_type === xactInitWriteUncached) + def needsMemWrite(a_type: UFix, global_state: UFix): Bool = { + (a_type === acquireWriteUncached) } - def needsAckReply(x_type: UFix, global_state: UFix): Bool = { - (x_type === xactInitWriteUncached) + def needsAckReply(a_type: UFix, global_state: UFix): Bool = { + (a_type === acquireWriteUncached) } } @@ -553,12 +553,12 @@ class MESICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(4){ UFix() } val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(3){ UFix() } - val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: Nil = Enum(7){ UFix() } - val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: Nil = Enum(8){ UFix() } - val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() } - val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() } - val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactInitReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) - val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(7){ UFix() } + val grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(8){ UFix() } + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(6){ UFix() } + val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) @@ -569,10 +569,10 @@ class MESICoherence extends CoherencePolicyWithUncached { state != tileInvalid } - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { val (read, write) = cpuCmdToRW(cmd) (read && messageIsUncached(outstanding)) || - (write && (outstanding.x_type != xactInitReadExclusive)) + (write && (outstanding.a_type != acquireReadExclusive)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( @@ -596,109 +596,109 @@ class MESICoherence extends CoherencePolicyWithUncached { } def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { - MuxLookup(incoming.x_type, tileInvalid, Array( - xactReplyReadShared -> tileShared, - xactReplyReadExclusive -> Mux(outstanding.x_type === xactInitReadExclusive, tileExclusiveDirty, tileExclusiveClean), - xactReplyReadExclusiveAck -> tileExclusiveDirty, - xactReplyReadUncached -> tileInvalid, - xactReplyWriteUncached -> tileInvalid, - xactReplyReadWordUncached -> tileInvalid, - xactReplyWriteWordUncached -> tileInvalid, - xactReplyAtomicUncached -> tileInvalid + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix = { + MuxLookup(incoming.g_type, tileInvalid, Array( + grantReadShared -> tileShared, + grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusive, tileExclusiveDirty, tileExclusiveClean), + grantReadExclusiveAck -> tileExclusiveDirty, + grantReadUncached -> tileInvalid, + grantWriteUncached -> tileInvalid, + grantReadWordUncached -> tileInvalid, + grantWriteWordUncached -> tileInvalid, + grantAtomicUncached -> tileInvalid )) } - def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { + def newStateOnProbe(incoming: Probe, state: UFix): Bits = { MuxLookup(incoming.p_type, state, Array( - probeReqInvalidate -> tileInvalid, - probeReqDowngrade -> tileShared, - probeReqCopy -> state + probeInvalidate -> tileInvalid, + probeDowngrade -> tileShared, + probeCopy -> state )) } - def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id) - def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id) - def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) - def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) - def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) - def isUncachedReadTransaction(xinit: TransactionInit) = xinit.x_type === xactInitReadUncached + def getUncachedReadAcquire(addr: UFix, id: UFix) = Acquire(acquireReadUncached, addr, id) + def getUncachedWriteAcquire(addr: UFix, id: UFix) = Acquire(acquireWriteUncached, addr, id) + def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) + def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) + def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) + def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached - def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, xactInitReadExclusive, xactInitReadShared) + Mux(write || cmd === M_PFW, acquireReadExclusive, acquireReadShared) } - def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, xactInitReadExclusive, outstanding.x_type) + Mux(write, acquireReadExclusive, outstanding.a_type) } - def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached - def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) + def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteUncached + def getAcquireTypeOnWriteback(): Bits = getAcquireTypeOnCacheControl(M_INV) - def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - val reply = new ProbeReply() - val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( - probeReqInvalidate -> probeRepInvalidateData, - probeReqDowngrade -> probeRepDowngradeData, - probeReqCopy -> probeRepCopyData + def newRelease (incoming: Probe, state: UFix): Release = { + val reply = new Release + val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( + probeInvalidate -> releaseInvalidateData, + probeDowngrade -> releaseDowngradeData, + probeCopy -> releaseCopyData )) - val without_data = MuxLookup(incoming.p_type, probeRepInvalidateAck, Array( - probeReqInvalidate -> probeRepInvalidateAck, - probeReqDowngrade -> probeRepDowngradeAck, - probeReqCopy -> probeRepCopyAck + val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> releaseInvalidateAck, + probeDowngrade -> releaseDowngradeAck, + probeCopy -> releaseCopyAck )) - reply.p_type := Mux(needsWriteback(state), with_data, without_data) - reply.global_xact_id := incoming.global_xact_id + reply.r_type := Mux(needsWriteback(state), with_data, without_data) + reply.master_xact_id := incoming.master_xact_id reply } - def messageHasData (reply: ProbeReply): Bool = { - (reply.p_type === probeRepInvalidateData || - reply.p_type === probeRepDowngradeData || - reply.p_type === probeRepCopyData) + def messageHasData (reply: Release): Bool = { + (reply.r_type === releaseInvalidateData || + reply.r_type === releaseDowngradeData || + reply.r_type === releaseCopyData) } - def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_) - def messageHasData (reply: TransactionReply): Bool = { - (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck && reply.x_type != xactReplyWriteWordUncached) + def messageHasData (acq: Acquire): Bool = hasDataTypeList.map(t => acq.a_type === t).reduceLeft(_||_) + def messageHasData (reply: Grant): Bool = { + (reply.g_type != grantWriteUncached && reply.g_type != grantReadExclusiveAck && reply.g_type != grantWriteWordUncached) } - def messageUpdatesDataArray (reply: TransactionReply): Bool = { - (reply.x_type === xactReplyReadShared || reply.x_type === xactReplyReadExclusive) + def messageUpdatesDataArray (reply: Grant): Bool = { + (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } - def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_) + def messageIsUncached(acq: Acquire): Bool = uncachedTypeList.map(t => acq.a_type === t).reduceLeft(_||_) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getTransactionReplyType(x_type: UFix, count: UFix): Bits = { - MuxLookup(x_type, xactReplyReadUncached, Array( - xactInitReadShared -> Mux(count > UFix(0), xactReplyReadShared, xactReplyReadExclusive), - xactInitReadExclusive -> xactReplyReadExclusive, - xactInitReadUncached -> xactReplyReadUncached, - xactInitWriteUncached -> xactReplyWriteUncached, - xactInitReadWordUncached -> xactReplyReadWordUncached, - xactInitWriteWordUncached -> xactReplyWriteWordUncached, - xactInitAtomicUncached -> xactReplyAtomicUncached + def getGrantType(a_type: UFix, count: UFix): Bits = { + MuxLookup(a_type, grantReadUncached, Array( + acquireReadShared -> Mux(count > UFix(0), grantReadShared, grantReadExclusive), + acquireReadExclusive -> grantReadExclusive, + acquireReadUncached -> grantReadUncached, + acquireWriteUncached -> grantWriteUncached, + acquireReadWordUncached -> grantReadWordUncached, + acquireWriteWordUncached -> grantWriteWordUncached, + acquireAtomicUncached -> grantAtomicUncached )) } - def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = { - MuxLookup(x_type, probeReqCopy, Array( - xactInitReadShared -> probeReqDowngrade, - xactInitReadExclusive -> probeReqInvalidate, - xactInitReadUncached -> probeReqCopy, - xactInitWriteUncached -> probeReqInvalidate, - xactInitReadWordUncached -> probeReqCopy, - xactInitWriteWordUncached -> probeReqInvalidate, - xactInitAtomicUncached -> probeReqInvalidate + def getProbeType(a_type: UFix, global_state: UFix): UFix = { + MuxLookup(a_type, probeCopy, Array( + acquireReadShared -> probeDowngrade, + acquireReadExclusive -> probeInvalidate, + acquireReadUncached -> probeCopy, + acquireWriteUncached -> probeInvalidate, + acquireReadWordUncached -> probeCopy, + acquireWriteWordUncached -> probeInvalidate, + acquireAtomicUncached -> probeInvalidate )) } - def needsMemRead(x_type: UFix, global_state: UFix): Bool = { - (x_type != xactInitWriteUncached) + def needsMemRead(a_type: UFix, global_state: UFix): Bool = { + (a_type != acquireWriteUncached) } - def needsMemWrite(x_type: UFix, global_state: UFix): Bool = { - (x_type === xactInitWriteUncached) + def needsMemWrite(a_type: UFix, global_state: UFix): Bool = { + (a_type === acquireWriteUncached) } - def needsAckReply(x_type: UFix, global_state: UFix): Bool = { - (x_type === xactInitWriteUncached) + def needsAckReply(a_type: UFix, global_state: UFix): Bool = { + (a_type === acquireWriteUncached) } } @@ -706,12 +706,12 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: tileSharedByTwo :: tileMigratoryClean :: tileMigratoryDirty :: Nil = Enum(7){ UFix() } - val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: xactInitInvalidateOthers :: Nil = Enum(8){ UFix() } - val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: xactReplyReadMigratory :: Nil = Enum(9){ UFix() } - val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: probeReqInvalidateOthers :: Nil = Enum(4){ UFix() } - val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: probeRepDowngradeDataMigratory :: probeRepDowngradeAckHasCopy :: probeRepInvalidateDataMigratory :: probeRepInvalidateAckMigratory :: Nil = Enum(10){ UFix() } - val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactInitReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) - val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(8){ UFix() } + val grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(9){ UFix() } + val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(4){ UFix() } + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(10){ UFix() } + val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) def uFixListContains(list: List[UFix], elem: UFix): Bool = list.map(elem === _).reduceLeft(_||_) @@ -723,10 +723,10 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { state != tileInvalid } - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { val (read, write) = cpuCmdToRW(cmd) (read && messageIsUncached(outstanding)) || - (write && (outstanding.x_type != xactInitReadExclusive && outstanding.x_type != xactInitInvalidateOthers)) + (write && (outstanding.a_type != acquireReadExclusive && outstanding.a_type != acquireInvalidateOthers)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( @@ -752,30 +752,30 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { - MuxLookup(incoming.x_type, tileInvalid, Array( - xactReplyReadShared -> tileShared, - xactReplyReadExclusive -> MuxLookup(outstanding.x_type, tileExclusiveDirty, Array( - xactInitReadExclusive -> tileExclusiveDirty, - xactInitReadShared -> tileExclusiveClean)), - xactReplyReadExclusiveAck -> tileExclusiveDirty, - xactReplyReadUncached -> tileInvalid, - xactReplyWriteUncached -> tileInvalid, - xactReplyReadWordUncached -> tileInvalid, - xactReplyWriteWordUncached -> tileInvalid, - xactReplyAtomicUncached -> tileInvalid, - xactReplyReadMigratory -> MuxLookup(outstanding.x_type, tileMigratoryDirty, Array( - xactInitInvalidateOthers -> tileMigratoryDirty, - xactInitReadExclusive -> tileMigratoryDirty, - xactInitReadShared -> tileMigratoryClean)) + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix = { + MuxLookup(incoming.g_type, tileInvalid, Array( + grantReadShared -> tileShared, + grantReadExclusive -> MuxLookup(outstanding.a_type, tileExclusiveDirty, Array( + acquireReadExclusive -> tileExclusiveDirty, + acquireReadShared -> tileExclusiveClean)), + grantReadExclusiveAck -> tileExclusiveDirty, + grantReadUncached -> tileInvalid, + grantWriteUncached -> tileInvalid, + grantReadWordUncached -> tileInvalid, + grantWriteWordUncached -> tileInvalid, + grantAtomicUncached -> tileInvalid, + grantReadMigratory -> MuxLookup(outstanding.a_type, tileMigratoryDirty, Array( + acquireInvalidateOthers -> tileMigratoryDirty, + acquireReadExclusive -> tileMigratoryDirty, + acquireReadShared -> tileMigratoryClean)) )) } - def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { + def newStateOnProbe(incoming: Probe, state: UFix): Bits = { MuxLookup(incoming.p_type, state, Array( - probeReqInvalidate -> tileInvalid, - probeReqInvalidateOthers -> tileInvalid, - probeReqCopy -> state, - probeReqDowngrade -> MuxLookup(state, tileShared, Array( + probeInvalidate -> tileInvalid, + probeInvalidateOthers -> tileInvalid, + probeCopy -> state, + probeDowngrade -> MuxLookup(state, tileShared, Array( tileExclusiveClean -> tileSharedByTwo, tileExclusiveDirty -> tileSharedByTwo, tileSharedByTwo -> tileShared, @@ -784,91 +784,91 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { )) } - def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id) - def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id) - def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) - def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) - def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) - def isUncachedReadTransaction(xinit: TransactionInit) = xinit.x_type === xactInitReadUncached + def getUncachedReadAcquire(addr: UFix, id: UFix) = Acquire(acquireReadUncached, addr, id) + def getUncachedWriteAcquire(addr: UFix, id: UFix) = Acquire(acquireWriteUncached, addr, id) + def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) + def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) + def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) + def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached - def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, Mux(state === tileInvalid, xactInitReadExclusive, xactInitInvalidateOthers), xactInitReadShared) + Mux(write || cmd === M_PFW, Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), acquireReadShared) } - def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, Mux(state === tileInvalid, xactInitReadExclusive, xactInitInvalidateOthers), outstanding.x_type) + Mux(write, Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), outstanding.a_type) } - def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached - def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) + def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteUncached + def getAcquireTypeOnWriteback(): Bits = getAcquireTypeOnCacheControl(M_INV) - def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - Assert( incoming.p_type === probeReqInvalidateOthers && needsWriteback(state), "Bad probe request type, should be impossible.") - val reply = new ProbeReply() - val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( - probeReqInvalidate -> Mux(uFixListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), - probeRepInvalidateDataMigratory, probeRepInvalidateData), - probeReqDowngrade -> Mux(state === tileMigratoryDirty, probeRepDowngradeDataMigratory, probeRepDowngradeData), - probeReqCopy -> probeRepCopyData + def newRelease (incoming: Probe, state: UFix): Release = { + Assert( incoming.p_type === probeInvalidateOthers && needsWriteback(state), "Bad probe request type, should be impossible.") + val reply = new Release() + val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( + probeInvalidate -> Mux(uFixListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), + releaseInvalidateDataMigratory, releaseInvalidateData), + probeDowngrade -> Mux(state === tileMigratoryDirty, releaseDowngradeDataMigratory, releaseDowngradeData), + probeCopy -> releaseCopyData )) - val without_data = MuxLookup(incoming.p_type, probeRepInvalidateAck, Array( - probeReqInvalidate -> Mux(tileExclusiveClean === state, probeRepInvalidateAckMigratory, probeRepInvalidateAck), - probeReqInvalidateOthers -> Mux(state === tileSharedByTwo, probeRepInvalidateAckMigratory, probeRepInvalidateAck), - probeReqDowngrade -> Mux(state != tileInvalid, probeRepDowngradeAckHasCopy, probeRepDowngradeAck), - probeReqCopy -> probeRepCopyAck + val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> Mux(tileExclusiveClean === state, releaseInvalidateAckMigratory, releaseInvalidateAck), + probeInvalidateOthers -> Mux(state === tileSharedByTwo, releaseInvalidateAckMigratory, releaseInvalidateAck), + probeDowngrade -> Mux(state != tileInvalid, releaseDowngradeAckHasCopy, releaseDowngradeAck), + probeCopy -> releaseCopyAck )) - reply.p_type := Mux(needsWriteback(state), with_data, without_data) - reply.global_xact_id := incoming.global_xact_id + reply.r_type := Mux(needsWriteback(state), with_data, without_data) + reply.master_xact_id := incoming.master_xact_id reply } - def messageHasData (reply: ProbeReply): Bool = { - uFixListContains(List(probeRepInvalidateData, probeRepDowngradeData, probeRepCopyData, probeRepInvalidateDataMigratory, probeRepDowngradeDataMigratory), reply.p_type) + def messageHasData (reply: Release): Bool = { + uFixListContains(List(releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory), reply.r_type) } - def messageHasData (init: TransactionInit): Bool = uFixListContains(hasDataTypeList, init.x_type) - def messageHasData (reply: TransactionReply): Bool = { - uFixListContains(List(xactReplyReadShared, xactReplyReadExclusive, xactReplyReadUncached, xactReplyReadMigratory, xactReplyReadWordUncached, xactReplyAtomicUncached), reply.x_type) + def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataTypeList, acq.a_type) + def messageHasData (reply: Grant): Bool = { + uFixListContains(List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadMigratory, grantReadWordUncached, grantAtomicUncached), reply.g_type) } - def messageUpdatesDataArray (reply: TransactionReply): Bool = { - uFixListContains(List(xactReplyReadShared, xactReplyReadExclusive, xactReplyReadMigratory), reply.x_type) + def messageUpdatesDataArray (reply: Grant): Bool = { + uFixListContains(List(grantReadShared, grantReadExclusive, grantReadMigratory), reply.g_type) } - def messageIsUncached(init: TransactionInit): Bool = uFixListContains(uncachedTypeList, init.x_type) + def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getTransactionReplyType(x_type: UFix, count: UFix): Bits = { - MuxLookup(x_type, xactReplyReadUncached, Array( - xactInitReadShared -> Mux(count > UFix(0), xactReplyReadShared, xactReplyReadExclusive), //TODO: what is count? Depend on probeRep.p_type??? - xactInitReadExclusive -> xactReplyReadExclusive, - xactInitReadUncached -> xactReplyReadUncached, - xactInitWriteUncached -> xactReplyWriteUncached, - xactInitReadWordUncached -> xactReplyReadWordUncached, - xactInitWriteWordUncached -> xactReplyWriteWordUncached, - xactInitAtomicUncached -> xactReplyAtomicUncached, - xactInitInvalidateOthers -> xactReplyReadExclusiveAck //TODO: add this to MESI? + def getGrantType(a_type: UFix, count: UFix): Bits = { + MuxLookup(a_type, grantReadUncached, Array( + acquireReadShared -> Mux(count > UFix(0), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? + acquireReadExclusive -> grantReadExclusive, + acquireReadUncached -> grantReadUncached, + acquireWriteUncached -> grantWriteUncached, + acquireReadWordUncached -> grantReadWordUncached, + acquireWriteWordUncached -> grantWriteWordUncached, + acquireAtomicUncached -> grantAtomicUncached, + acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI? )) } - def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = { - MuxLookup(x_type, probeReqCopy, Array( - xactInitReadShared -> probeReqDowngrade, - xactInitReadExclusive -> probeReqInvalidate, - xactInitReadUncached -> probeReqCopy, - xactInitWriteUncached -> probeReqInvalidate, - xactInitReadWordUncached -> probeReqCopy, - xactInitWriteWordUncached -> probeReqInvalidate, - xactInitAtomicUncached -> probeReqInvalidate, - xactInitInvalidateOthers -> probeReqInvalidateOthers + def getProbeType(a_type: UFix, global_state: UFix): UFix = { + MuxLookup(a_type, probeCopy, Array( + acquireReadShared -> probeDowngrade, + acquireReadExclusive -> probeInvalidate, + acquireReadUncached -> probeCopy, + acquireWriteUncached -> probeInvalidate, + acquireReadWordUncached -> probeCopy, + acquireWriteWordUncached -> probeInvalidate, + acquireAtomicUncached -> probeInvalidate, + acquireInvalidateOthers -> probeInvalidateOthers )) } - def needsMemRead(x_type: UFix, global_state: UFix): Bool = { - (x_type != xactInitWriteUncached && x_type != xactInitInvalidateOthers) + def needsMemRead(a_type: UFix, global_state: UFix): Bool = { + (a_type != acquireWriteUncached && a_type != acquireInvalidateOthers) } - def needsMemWrite(x_type: UFix, global_state: UFix): Bool = { - (x_type === xactInitWriteUncached || x_type === xactInitWriteWordUncached || x_type === xactInitAtomicUncached) + def needsMemWrite(a_type: UFix, global_state: UFix): Bool = { + (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached || a_type === acquireAtomicUncached) } - def needsAckReply(x_type: UFix, global_state: UFix): Bool = { - (x_type === xactInitWriteUncached || x_type === xactInitWriteWordUncached ||x_type === xactInitInvalidateOthers) + def needsAckReply(a_type: UFix, global_state: UFix): Bool = { + (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached ||a_type === acquireInvalidateOthers) } } diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index 0b885352..b515c1c8 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -11,7 +11,7 @@ abstract trait CoherenceConfigConstants { trait UncoreConstants { val NGLOBAL_XACTS = 8 - val GLOBAL_XACT_ID_BITS = log2Up(NGLOBAL_XACTS) + val MASTER_XACT_ID_BITS = log2Up(NGLOBAL_XACTS) val CACHE_DATA_SIZE_IN_BYTES = 1 << 6 } @@ -20,19 +20,19 @@ trait CacheConstants extends UncoreConstants { } trait TileLinkTypeConstants { - val X_INIT_TYPE_MAX_BITS = 2 - val X_REP_TYPE_MAX_BITS = 3 - val P_REQ_TYPE_MAX_BITS = 2 - val P_REP_TYPE_MAX_BITS = 3 + val ACQUIRE_TYPE_MAX_BITS = 2 + val GRANT_TYPE_MAX_BITS = 3 + val PROBE_TYPE_MAX_BITS = 2 + val RELEASE_TYPE_MAX_BITS = 3 } trait TileLinkSizeConstants extends TileLinkTypeConstants { - val TILE_XACT_ID_BITS = 5 - val X_INIT_WRITE_MASK_BITS = 6 - val X_INIT_SUBWORD_ADDR_BITS = 3 - val X_INIT_ATOMIC_OP_BITS = 4 + val CLIENT_XACT_ID_BITS = 5 + val ACQUIRE_WRITE_MASK_BITS = 6 + val ACQUIRE_SUBWORD_ADDR_BITS = 3 + val ACQUIRE_ATOMIC_OP_BITS = 4 } trait MemoryOpConstants { @@ -72,7 +72,7 @@ trait MemoryInterfaceConstants extends UncoreConstants with TileLinkSizeConstants { - val MEM_TAG_BITS = max(TILE_XACT_ID_BITS, GLOBAL_XACT_ID_BITS) + val MEM_TAG_BITS = max(CLIENT_XACT_ID_BITS, MASTER_XACT_ID_BITS) val MEM_DATA_BITS = 128 val REFILL_CYCLES = CACHE_DATA_SIZE_IN_BYTES*8/MEM_DATA_BITS } diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 2e73a6d7..0e69ff47 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -110,14 +110,14 @@ class ClientSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data class MasterSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) {flip()} class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val xact_init = (new ClientSourcedIO){(new LogicalNetworkIO){new TransactionInit }} - val xact_init_data = (new ClientSourcedIO){(new LogicalNetworkIO){new TransactionInitData }} - val xact_abort = (new MasterSourcedIO){(new LogicalNetworkIO){new TransactionAbort }} - val probe_req = (new MasterSourcedIO){(new LogicalNetworkIO){new ProbeRequest }} - val probe_rep = (new ClientSourcedIO){(new LogicalNetworkIO){new ProbeReply }} - val probe_rep_data = (new ClientSourcedIO){(new LogicalNetworkIO){new ProbeReplyData }} - val xact_rep = (new MasterSourcedIO){(new LogicalNetworkIO){new TransactionReply }} - val xact_finish = (new ClientSourcedIO){(new LogicalNetworkIO){new TransactionFinish }} + val acquire = (new ClientSourcedIO){(new LogicalNetworkIO){new Acquire }} + val acquire_data = (new ClientSourcedIO){(new LogicalNetworkIO){new AcquireData }} + val abort = (new MasterSourcedIO){(new LogicalNetworkIO){new Abort }} + val probe = (new MasterSourcedIO){(new LogicalNetworkIO){new Probe }} + val release = (new ClientSourcedIO){(new LogicalNetworkIO){new Release }} + val release_data = (new ClientSourcedIO){(new LogicalNetworkIO){new ReleaseData }} + val grant = (new MasterSourcedIO){(new LogicalNetworkIO){new Grant }} + val grant_ack = (new ClientSourcedIO){(new LogicalNetworkIO){new GrantAck }} override def clone = { new TileLinkIO().asInstanceOf[this.type] } } } diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index f07e7ed1..8ff26adc 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -32,81 +32,83 @@ class ioMemPipe extends Bundle { val resp = (new PipeIO) { new MemResp() }.flip } -class TransactionInit extends PhysicalAddress { - val x_type = Bits(width = X_INIT_TYPE_MAX_BITS) - val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) - val write_mask = Bits(width = X_INIT_WRITE_MASK_BITS) - val subword_addr = Bits(width = X_INIT_SUBWORD_ADDR_BITS) - val atomic_opcode = Bits(width = X_INIT_ATOMIC_OP_BITS) +class Acquire extends PhysicalAddress { + val a_type = Bits(width = ACQUIRE_TYPE_MAX_BITS) + val client_xact_id = Bits(width = CLIENT_XACT_ID_BITS) + val write_mask = Bits(width = ACQUIRE_WRITE_MASK_BITS) + val subword_addr = Bits(width = ACQUIRE_SUBWORD_ADDR_BITS) + val atomic_opcode = Bits(width = ACQUIRE_ATOMIC_OP_BITS) } -object TransactionInit +object Acquire { - def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix) = { - val init = new TransactionInit - init.x_type := x_type - init.addr := addr - init.tile_xact_id := tile_xact_id - init + def apply(a_type: Bits, addr: UFix, client_xact_id: UFix) = { + val acq = new Acquire + acq.a_type := a_type + acq.addr := addr + acq.client_xact_id := client_xact_id + acq } - def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix, write_mask: Bits) = { - val init = new TransactionInit - init.x_type := x_type - init.addr := addr - init.tile_xact_id := tile_xact_id - init.write_mask := write_mask - init + def apply(a_type: Bits, addr: UFix, client_xact_id: UFix, write_mask: Bits) = { + val acq = new Acquire + acq.a_type := a_type + acq.addr := addr + acq.client_xact_id := client_xact_id + acq.write_mask := write_mask + acq } - def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix, subword_addr: UFix, atomic_opcode: UFix) = { - val init = new TransactionInit - init.x_type := x_type - init.addr := addr - init.tile_xact_id := tile_xact_id - init.subword_addr := subword_addr - init.atomic_opcode := atomic_opcode - init + def apply(a_type: Bits, addr: UFix, client_xact_id: UFix, subword_addr: UFix, atomic_opcode: UFix) = { + val acq = new Acquire + acq.a_type := a_type + acq.addr := addr + acq.client_xact_id := client_xact_id + acq.subword_addr := subword_addr + acq.atomic_opcode := atomic_opcode + acq } } -class TransactionInitData extends MemData +class AcquireData extends MemData -class TransactionAbort extends Bundle { - val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) +class Abort extends Bundle { + val client_xact_id = Bits(width = CLIENT_XACT_ID_BITS) } -class ProbeRequest extends PhysicalAddress { - val p_type = Bits(width = P_REQ_TYPE_MAX_BITS) - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) +class Probe extends PhysicalAddress { + val p_type = Bits(width = PROBE_TYPE_MAX_BITS) + val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) } -class ProbeReply extends Bundle { - val p_type = Bits(width = P_REP_TYPE_MAX_BITS) - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) +class Release extends Bundle { + val r_type = Bits(width = RELEASE_TYPE_MAX_BITS) + val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) } -class ProbeReplyData extends MemData +class ReleaseData extends MemData -class TransactionReply extends MemData { - val x_type = Bits(width = X_REP_TYPE_MAX_BITS) - val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) +class Grant extends MemData { + val g_type = Bits(width = GRANT_TYPE_MAX_BITS) + val client_xact_id = Bits(width = CLIENT_XACT_ID_BITS) + val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) val require_ack = Bool() } -class TransactionFinish extends Bundle { - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) +class GrantAck extends Bundle { + val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) } -/* -class ioTileLink extends Bundle { - val xact_init = (new FIFOIO) { new TransactionInit } - val xact_init_data = (new FIFOIO) { new TransactionInitData } - val xact_abort = (new FIFOIO) { new TransactionAbort }.flip - val probe_req = (new FIFOIO) { new ProbeRequest }.flip - val probe_rep = (new FIFOIO) { new ProbeReply } - val probe_rep_data = (new FIFOIO) { new ProbeReplyData } - val xact_rep = (new FIFOIO) { new TransactionReply }.flip - val xact_finish = (new FIFOIO) { new TransactionFinish } - val incoherent = Bool(OUTPUT) +abstract class DirectionalFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) +class ClientSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) +class MasterSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) {flip()} + +class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val acquire = (new ClientSourcedIO){(new LogicalNetworkIO){new Acquire }} + val acquire_data = (new ClientSourcedIO){(new LogicalNetworkIO){new AcquireData }} + val abort = (new MasterSourcedIO){(new LogicalNetworkIO){new Abort }} + val probe = (new MasterSourcedIO){(new LogicalNetworkIO){new Probe }} + val release = (new ClientSourcedIO){(new LogicalNetworkIO){new Release }} + val release_data = (new ClientSourcedIO){(new LogicalNetworkIO){new ReleaseData }} + val grant = (new MasterSourcedIO){(new LogicalNetworkIO){new Grant }} + val grant_ack = (new ClientSourcedIO){(new LogicalNetworkIO){new GrantAck }} + override def clone = { new TileLinkIO().asInstanceOf[this.type] } } -*/ diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 179179e8..690454a1 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -8,13 +8,13 @@ class TrackerProbeData(implicit conf: CoherenceHubConfiguration) extends Bundle } class TrackerAllocReq(implicit conf: CoherenceHubConfiguration) extends Bundle { - val xact_init = new TransactionInit() + val acquire = new Acquire() val client_id = Bits(width = conf.ln.idBits) override def clone = { new TrackerAllocReq().asInstanceOf[this.type] } } class TrackerDependency extends Bundle { - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) + val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) } class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) extends Component { @@ -23,27 +23,27 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val alloc_req = (new FIFOIO) { new TrackerAllocReq }.flip val p_data = (new PipeIO) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) - val xact_finish = Bool(INPUT) + val grant_ack = Bool(INPUT) val p_rep_cnt_dec = Bits(INPUT, conf.ln.nTiles) val p_req_cnt_inc = Bits(INPUT, conf.ln.nTiles) val tile_incoherent = Bits(INPUT, conf.ln.nTiles) - val p_rep_data = (new PipeIO) { new ProbeReplyData }.flip - val x_init_data = (new PipeIO) { new TransactionInitData }.flip - val sent_x_rep_ack = Bool(INPUT) + val p_rep_data = (new PipeIO) { new ReleaseData }.flip + val x_init_data = (new PipeIO) { new AcquireData }.flip + val sent_grant_ack = Bool(INPUT) val p_rep_data_dep = (new PipeIO) { new TrackerDependency }.flip val x_init_data_dep = (new PipeIO) { new TrackerDependency }.flip val mem_req_cmd = (new FIFOIO) { new MemReqCmd } val mem_req_data = (new FIFOIO) { new MemData } val mem_req_lock = Bool(OUTPUT) - val probe_req = (new FIFOIO) { new ProbeRequest } + val probe = (new FIFOIO) { new Probe } val busy = Bool(OUTPUT) val addr = Bits(OUTPUT, PADDR_BITS - OFFSET_BITS) val init_client_id = Bits(OUTPUT, conf.ln.idBits) val p_rep_client_id = Bits(OUTPUT, conf.ln.idBits) - val tile_xact_id = Bits(OUTPUT, TILE_XACT_ID_BITS) + val client_xact_id = Bits(OUTPUT, CLIENT_XACT_ID_BITS) val sharer_count = Bits(OUTPUT, conf.ln.idBits+1) - val x_type = Bits(OUTPUT, X_INIT_TYPE_MAX_BITS) + val a_type = Bits(OUTPUT, ACQUIRE_TYPE_MAX_BITS) val push_p_req = Bits(OUTPUT, conf.ln.nTiles) val pop_p_rep = Bits(OUTPUT, conf.ln.nTiles) val pop_p_rep_data = Bits(OUTPUT, conf.ln.nTiles) @@ -51,7 +51,7 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val pop_x_init = Bits(OUTPUT, conf.ln.nTiles) val pop_x_init_data = Bits(OUTPUT, conf.ln.nTiles) val pop_x_init_dep = Bits(OUTPUT, conf.ln.nTiles) - val send_x_rep_ack = Bool(OUTPUT) + val send_grant_ack = Bool(OUTPUT) } def doMemReqWrite(req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: PipeIO[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, client_id: UFix) { @@ -89,7 +89,7 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) - val xact = Reg{ new TransactionInit } + val xact = Reg{ new Acquire } val init_client_id_ = Reg{ Bits() } val p_rep_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) val p_req_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) @@ -108,7 +108,7 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex // issue self-probes for uncached read xacts to facilitate I$ coherence // TODO: this is hackish; figure out how to do it more systematically val probe_self = co match { - case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.alloc_req.bits.xact_init) + case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.alloc_req.bits.acquire) case _ => Bool(false) } val myflag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.client_id(log2Up(conf.ln.nTiles)-1,0))) @@ -119,9 +119,9 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex io.addr := xact.addr io.init_client_id := init_client_id_ io.p_rep_client_id := p_rep_client_id_ - io.tile_xact_id := xact.tile_xact_id + io.client_xact_id := xact.client_xact_id io.sharer_count := UFix(conf.ln.nTiles) // TODO: Broadcast only - io.x_type := xact.x_type + io.a_type := xact.a_type io.mem_req_cmd.valid := Bool(false) io.mem_req_cmd.bits.rw := Bool(false) @@ -130,10 +130,10 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex io.mem_req_data.valid := Bool(false) io.mem_req_data.bits.data := UFix(0) io.mem_req_lock := Bool(false) - io.probe_req.valid := Bool(false) - io.probe_req.bits.p_type := co.getProbeRequestType(xact.x_type, UFix(0)) - io.probe_req.bits.global_xact_id := UFix(id) - io.probe_req.bits.addr := xact.addr + io.probe.valid := Bool(false) + io.probe.bits.p_type := co.getProbeType(xact.a_type, UFix(0)) + io.probe.bits.master_xact_id := UFix(id) + io.probe.bits.addr := xact.addr io.push_p_req := Bits(0, width = conf.ln.nTiles) io.pop_p_rep := Bits(0, width = conf.ln.nTiles) io.pop_p_rep_data := Bits(0, width = conf.ln.nTiles) @@ -141,15 +141,15 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex io.pop_x_init := Bits(0, width = conf.ln.nTiles) io.pop_x_init_data := Bits(0, width = conf.ln.nTiles) io.pop_x_init_dep := Bits(0, width = conf.ln.nTiles) - io.send_x_rep_ack := Bool(false) + io.send_grant_ack := Bool(false) switch (state) { is(s_idle) { when( io.alloc_req.valid && io.can_alloc ) { - xact := io.alloc_req.bits.xact_init + xact := io.alloc_req.bits.acquire init_client_id_ := io.alloc_req.bits.client_id - x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) - x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) + x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.acquire) + x_needs_read := co.needsMemRead(io.alloc_req.bits.acquire.a_type, UFix(0)) p_req_flags := p_req_initial_flags mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) @@ -164,7 +164,7 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex is(s_probe) { when(p_req_flags.orR) { io.push_p_req := p_req_flags - io.probe_req.valid := Bool(true) + io.probe.valid := Bool(true) } when(io.p_req_cnt_inc.orR) { p_req_flags := p_req_flags & ~io.p_req_cnt_inc // unflag sent reqs @@ -192,7 +192,7 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex p_w_mem_cmd_sent, io.pop_p_rep_data, io.pop_p_rep_dep, - io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id)), + io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.master_xact_id === UFix(id)), p_rep_client_id_) } . elsewhen(x_init_data_needs_write) { doMemReqWrite(io.mem_req_cmd, @@ -203,20 +203,20 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex x_w_mem_cmd_sent, io.pop_x_init_data, io.pop_x_init_dep, - io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id)), + io.x_init_data_dep.valid && (io.x_init_data_dep.bits.master_xact_id === UFix(id)), init_client_id_) } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { - state := Mux(co.needsAckReply(xact.x_type, UFix(0)), s_ack, s_busy) + state := Mux(co.needsAckReply(xact.a_type, UFix(0)), s_ack, s_busy) } } is(s_ack) { - io.send_x_rep_ack := Bool(true) - when(io.sent_x_rep_ack) { state := s_busy } + io.send_grant_ack := Bool(true) + when(io.sent_grant_ack) { state := s_busy } } is(s_busy) { // Nothing left to do but wait for transaction to complete - when (io.xact_finish) { + when (io.grant_ack) { state := s_idle } } @@ -274,28 +274,28 @@ class CoherenceHubNull(implicit conf: CoherenceHubConfiguration) extends Coheren { val co = conf.co.asInstanceOf[ThreeStateIncoherence] - val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.payload.x_type === co.xactInitWriteback + val x_init = io.tiles(0).acquire + val is_write = x_init.bits.payload.a_type === co.acquireWriteback x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write - io.mem.req_cmd.bits.tag := x_init.bits.payload.tile_xact_id + io.mem.req_cmd.bits.tag := x_init.bits.payload.client_xact_id io.mem.req_cmd.bits.addr := x_init.bits.payload.addr - io.mem.req_data <> io.tiles(0).xact_init_data + io.mem.req_data <> io.tiles(0).acquire_data - val x_rep = io.tiles(0).xact_rep - x_rep.bits.payload.x_type := Mux(io.mem.resp.valid, co.xactReplyData, co.xactReplyAck) - x_rep.bits.payload.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.payload.tile_xact_id) - x_rep.bits.payload.global_xact_id := UFix(0) // don't care - x_rep.bits.payload.data := io.mem.resp.bits.data - x_rep.bits.payload.require_ack := Bool(true) - x_rep.valid := io.mem.resp.valid || x_init.valid && is_write && io.mem.req_cmd.ready + val grant = io.tiles(0).grant + grant.bits.payload.g_type := Mux(io.mem.resp.valid, co.grantData, co.grantAck) + grant.bits.payload.client_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.payload.client_xact_id) + grant.bits.payload.master_xact_id := UFix(0) // don't care + grant.bits.payload.data := io.mem.resp.bits.data + grant.bits.payload.require_ack := Bool(true) + grant.valid := io.mem.resp.valid || x_init.valid && is_write && io.mem.req_cmd.ready - io.tiles(0).xact_abort.valid := Bool(false) - io.tiles(0).xact_finish.ready := Bool(true) - io.tiles(0).probe_req.valid := Bool(false) - io.tiles(0).probe_rep.ready := Bool(true) - io.tiles(0).probe_rep_data.ready := Bool(true) + io.tiles(0).abort.valid := Bool(false) + io.tiles(0).grant_ack.ready := Bool(true) + io.tiles(0).probe.valid := Bool(false) + io.tiles(0).release.ready := Bool(true) + io.tiles(0).release_data.ready := Bool(true) } @@ -308,15 +308,15 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val busy_arr = Vec(NGLOBAL_XACTS){ Bool() } val addr_arr = Vec(NGLOBAL_XACTS){ Bits(width=PADDR_BITS-OFFSET_BITS) } val init_client_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } - val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_XACT_ID_BITS) } - val x_type_arr = Vec(NGLOBAL_XACTS){ Bits(width=X_INIT_TYPE_MAX_BITS) } + val client_xact_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=CLIENT_XACT_ID_BITS) } + val a_type_arr = Vec(NGLOBAL_XACTS){ Bits(width=ACQUIRE_TYPE_MAX_BITS) } val sh_count_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } - val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } + val send_grant_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } val do_free_arr = Vec(NGLOBAL_XACTS){ Bool() } val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } - val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } + val sent_grant_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } val p_data_client_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } @@ -325,19 +325,19 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co busy_arr(i) := t.busy addr_arr(i) := t.addr init_client_id_arr(i) := t.init_client_id - tile_xact_id_arr(i) := t.tile_xact_id - x_type_arr(i) := t.x_type + client_xact_id_arr(i) := t.client_xact_id + a_type_arr(i) := t.a_type sh_count_arr(i) := t.sharer_count - send_x_rep_ack_arr(i) := t.send_x_rep_ack - t.xact_finish := do_free_arr(i) + send_grant_ack_arr(i) := t.send_grant_ack + t.grant_ack := do_free_arr(i) t.p_data.bits.client_id := p_data_client_id_arr(i) t.p_data.valid := p_data_valid_arr(i) t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits t.tile_incoherent := io.incoherent.toBits - t.sent_x_rep_ack := sent_x_rep_ack_arr(i) + t.sent_grant_ack := sent_grant_ack_arr(i) do_free_arr(i) := Bool(false) - sent_x_rep_ack_arr(i) := Bool(false) + sent_grant_ack_arr(i) := Bool(false) p_data_client_id_arr(i) := Bits(0, width = conf.ln.idBits) p_data_valid_arr(i) := Bool(false) for( j <- 0 until conf.ln.nTiles) { @@ -351,9 +351,9 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co // Free finished transactions for( j <- 0 until conf.ln.nTiles ) { - val finish = io.tiles(j).xact_finish + val finish = io.tiles(j).grant_ack when (finish.valid) { - do_free_arr(finish.bits.payload.global_xact_id) := Bool(true) + do_free_arr(finish.bits.payload.master_xact_id) := Bool(true) } finish.ready := Bool(true) } @@ -361,31 +361,31 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co // Reply to initial requestor // Forward memory responses from mem to tile or arbitrate to ack val mem_idx = io.mem.resp.bits.tag - val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) + val ack_idx = PriorityEncoder(send_grant_ack_arr.toBits) for( j <- 0 until conf.ln.nTiles ) { - val rep = io.tiles(j).xact_rep - rep.bits.payload.x_type := UFix(0) - rep.bits.payload.tile_xact_id := UFix(0) - rep.bits.payload.global_xact_id := UFix(0) + val rep = io.tiles(j).grant + rep.bits.payload.g_type := UFix(0) + rep.bits.payload.client_xact_id := UFix(0) + rep.bits.payload.master_xact_id := UFix(0) rep.bits.payload.data := io.mem.resp.bits.data rep.bits.payload.require_ack := Bool(true) rep.valid := Bool(false) when(io.mem.resp.valid && (UFix(j) === init_client_id_arr(mem_idx))) { - rep.bits.payload.x_type := co.getTransactionReplyType(x_type_arr(mem_idx), sh_count_arr(mem_idx)) - rep.bits.payload.tile_xact_id := tile_xact_id_arr(mem_idx) - rep.bits.payload.global_xact_id := mem_idx + rep.bits.payload.g_type := co.getGrantType(a_type_arr(mem_idx), sh_count_arr(mem_idx)) + rep.bits.payload.client_xact_id := client_xact_id_arr(mem_idx) + rep.bits.payload.master_xact_id := mem_idx rep.valid := Bool(true) } . otherwise { - rep.bits.payload.x_type := co.getTransactionReplyType(x_type_arr(ack_idx), sh_count_arr(ack_idx)) - rep.bits.payload.tile_xact_id := tile_xact_id_arr(ack_idx) - rep.bits.payload.global_xact_id := ack_idx + rep.bits.payload.g_type := co.getGrantType(a_type_arr(ack_idx), sh_count_arr(ack_idx)) + rep.bits.payload.client_xact_id := client_xact_id_arr(ack_idx) + rep.bits.payload.master_xact_id := ack_idx when (UFix(j) === init_client_id_arr(ack_idx)) { - rep.valid := send_x_rep_ack_arr.toBits.orR - sent_x_rep_ack_arr(ack_idx) := rep.ready + rep.valid := send_grant_ack_arr.toBits.orR + sent_grant_ack_arr(ack_idx) := rep.ready } } } - io.mem.resp.ready := io.tiles(init_client_id_arr(mem_idx)).xact_rep.ready + io.mem.resp.ready := io.tiles(init_client_id_arr(mem_idx)).grant.ready // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests @@ -402,14 +402,14 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co // Handle probe replies, which may or may not have data for( j <- 0 until conf.ln.nTiles ) { - val p_rep = io.tiles(j).probe_rep - val p_rep_data = io.tiles(j).probe_rep_data - val idx = p_rep.bits.payload.global_xact_id + val p_rep = io.tiles(j).release + val p_rep_data = io.tiles(j).release_data + val idx = p_rep.bits.payload.master_xact_id val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) val do_pop = foldR(pop_p_reps)(_ || _) p_rep.ready := Bool(true) p_rep_data_dep_list(j).io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits.payload) - p_rep_data_dep_list(j).io.enq.bits.global_xact_id := p_rep.bits.payload.global_xact_id + p_rep_data_dep_list(j).io.enq.bits.master_xact_id := p_rep.bits.payload.master_xact_id p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) when (p_rep.valid && co.messageHasData(p_rep.bits.payload)) { p_data_valid_arr(idx) := Bool(true) @@ -418,15 +418,15 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) } for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_client_id).probe_rep_data.valid - trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_client_id).probe_rep_data.bits.payload + trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_client_id).release_data.valid + trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_client_id).release_data.bits.payload trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_client_id, p_rep_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_client_id, p_rep_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) for( j <- 0 until conf.ln.nTiles) { - val p_rep = io.tiles(j).probe_rep - p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.payload.global_xact_id === UFix(i)) + val p_rep = io.tiles(j).release + p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.payload.master_xact_id === UFix(i)) } } @@ -435,16 +435,16 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val abort_state_arr = Vec(conf.ln.nTiles) { Reg(resetVal = s_idle) } val want_to_abort_arr = Vec(conf.ln.nTiles) { Bool() } for( j <- 0 until conf.ln.nTiles ) { - val x_init = io.tiles(j).xact_init - val x_init_data = io.tiles(j).xact_init_data - val x_abort = io.tiles(j).xact_abort + val x_init = io.tiles(j).acquire + val x_init_data = io.tiles(j).acquire_data + val x_abort = io.tiles(j).abort val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val conflicts = Vec(NGLOBAL_XACTS) { Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.payload.addr) } - x_abort.bits.payload.tile_xact_id := x_init.bits.payload.tile_xact_id + x_abort.bits.payload.client_xact_id := x_init.bits.payload.client_xact_id want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits.payload))) x_abort.valid := Bool(false) @@ -486,23 +486,23 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co trackerList(i).io.alloc_req.bits := init_arb.io.out.bits trackerList(i).io.alloc_req.valid := init_arb.io.out.valid - trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_client_id).xact_init_data.bits.payload - trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_client_id).xact_init_data.valid + trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_client_id).acquire_data.bits.payload + trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_client_id).acquire_data.valid trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_client_id, x_init_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_client_id, x_init_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) } for( j <- 0 until conf.ln.nTiles ) { - val x_init = io.tiles(j).xact_init - val x_init_data = io.tiles(j).xact_init_data + val x_init = io.tiles(j).acquire + val x_init_data = io.tiles(j).acquire_data val x_init_data_dep = x_init_data_dep_list(j).io.deq - val x_abort = io.tiles(j).xact_abort + val x_abort = io.tiles(j).abort init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid - init_arb.io.in(j).bits.xact_init := x_init.bits.payload + init_arb.io.in(j).bits.acquire := x_init.bits.payload init_arb.io.in(j).bits.client_id := UFix(j) val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) val do_pop = foldR(pop_x_inits)(_||_) x_init_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(x_init.bits.payload) && (abort_state_arr(j) === s_idle) - x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) + x_init_data_dep_list(j).io.enq.bits.master_xact_id := OHToUFix(pop_x_inits) x_init.ready := (x_abort.valid && x_abort.ready) || do_pop x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) x_init_data_dep.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) @@ -512,15 +512,15 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co // Handle probe request generation // Must arbitrate for each request port - val p_req_arb_arr = List.fill(conf.ln.nTiles)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) + val p_req_arb_arr = List.fill(conf.ln.nTiles)((new Arbiter(NGLOBAL_XACTS)) { new Probe() }) for( j <- 0 until conf.ln.nTiles ) { for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io - p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits - p_req_arb_arr(j).io.in(i).valid := t.probe_req.valid && t.push_p_req(j) + p_req_arb_arr(j).io.in(i).bits := t.probe.bits + p_req_arb_arr(j).io.in(i).valid := t.probe.valid && t.push_p_req(j) p_req_cnt_inc_arr(i)(j) := p_req_arb_arr(j).io.in(i).ready } - FIFOedLogicalNetworkIOWrapper(p_req_arb_arr(j).io.out) <> io.tiles(j).probe_req + FIFOedLogicalNetworkIOWrapper(p_req_arb_arr(j).io.out) <> io.tiles(j).probe } } @@ -552,9 +552,9 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren // Handle transaction initiation requests // Only one allocation per cycle // Init requests may or may not have data - val x_init = io.network.xact_init - val x_init_data = io.network.xact_init_data - val x_abort = io.network.xact_abort + val x_init = io.network.acquire + val x_init_data = io.network.acquire_data + val x_abort = io.network.abort val x_dep_deq = x_init_data_dep_q.io.deq val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } val abort_state = Reg(resetVal = s_idle) @@ -578,14 +578,14 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren x_init.ready := (x_abort.valid && x_abort.ready) || pop_x_init x_init_data.ready := (abort_state === s_abort_drain) || trackerList.map(_.io.x_init_data.ready).reduce(_||_) x_init_data_dep_q.io.enq.valid := pop_x_init && co.messageHasData(x_init.bits.payload) && (abort_state === s_idle) - x_init_data_dep_q.io.enq.bits.global_xact_id := OHToUFix(trackerList.map(_.io.x_init.ready)) + x_init_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(trackerList.map(_.io.x_init.ready)) x_dep_deq.ready := trackerList.map(_.io.x_init_data_dep.ready).reduce(_||_) alloc_arb.io.out.ready := x_init.valid // Nack conflicting transaction init attempts x_abort.bits.header.dst := x_init.bits.header.src - x_abort.bits.payload.tile_xact_id := x_init.bits.payload.tile_xact_id + x_abort.bits.payload.client_xact_id := x_init.bits.payload.client_xact_id x_abort.valid := Bool(false) switch(abort_state) { is(s_idle) { @@ -610,23 +610,23 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren } // Handle probe request generation - val p_req_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new ProbeRequest }} + val p_req_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new Probe }} for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io p_req_arb.io.in(i).bits := t.p_req.bits p_req_arb.io.in(i).valid := t.p_req.valid t.p_req.ready := p_req_arb.io.in(i).ready } - io.network.probe_req <> p_req_arb.io.out + io.network.probe <> p_req_arb.io.out // Handle probe replies, which may or may not have data - val p_rep = io.network.probe_rep - val p_rep_data = io.network.probe_rep_data - val idx = p_rep.bits.payload.global_xact_id + val p_rep = io.network.release + val p_rep_data = io.network.release_data + val idx = p_rep.bits.payload.master_xact_id p_rep.ready := trackerList.map(_.io.p_rep.ready).reduce(_||_) p_rep_data.ready := trackerList.map(_.io.p_rep_data.ready).reduce(_||_) p_rep_data_dep_q.io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits.payload) - p_rep_data_dep_q.io.enq.bits.global_xact_id := p_rep.bits.payload.global_xact_id + p_rep_data_dep_q.io.enq.bits.master_xact_id := p_rep.bits.payload.master_xact_id p_rep_data_dep_q.io.deq.ready := trackerList.map(_.io.p_rep_data_dep.ready).reduce(_||_) for( i <- 0 until NGLOBAL_XACTS ) { trackerList(i).io.p_rep_data.valid := p_rep_data.valid @@ -639,29 +639,29 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren // Reply to initial requestor // Forward memory responses from mem to tile or arbitrate to ack - val x_rep_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new TransactionReply }} + val grant_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new Grant }} for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io - x_rep_arb.io.in(i).bits := t.x_rep.bits - x_rep_arb.io.in(i).valid := t.x_rep.valid - t.x_rep.ready := x_rep_arb.io.in(i).ready + grant_arb.io.in(i).bits := t.grant.bits + grant_arb.io.in(i).valid := t.grant.valid + t.grant.ready := grant_arb.io.in(i).ready } - x_rep_arb.io.out.ready := Bool(false) - io.network.xact_rep.valid := x_rep_arb.io.out.valid - io.network.xact_rep.bits := x_rep_arb.io.out.bits - x_rep_arb.io.out.ready := io.network.xact_rep.ready + grant_arb.io.out.ready := Bool(false) + io.network.grant.valid := grant_arb.io.out.valid + io.network.grant.bits := grant_arb.io.out.bits + grant_arb.io.out.ready := io.network.grant.ready when(io.mem.resp.valid) { - io.network.xact_rep.valid := Bool(true) - io.network.xact_rep.bits := Vec(trackerList.map(_.io.x_rep.bits)){(new LogicalNetworkIO){new TransactionReply}}(io.mem.resp.bits.tag) + io.network.grant.valid := Bool(true) + io.network.grant.bits := Vec(trackerList.map(_.io.grant.bits)){(new LogicalNetworkIO){new Grant}}(io.mem.resp.bits.tag) for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.x_rep.ready := (io.mem.resp.bits.tag === UFix(i)) && io.network.xact_rep.ready + trackerList(i).io.grant.ready := (io.mem.resp.bits.tag === UFix(i)) && io.network.grant.ready } } // Free finished transactions - val finish = io.network.xact_finish + val finish = io.network.grant_ack for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.free := finish.valid && (finish.bits.payload.global_xact_id === UFix(i)) + trackerList(i).io.free := finish.valid && (finish.bits.payload.master_xact_id === UFix(i)) } finish.ready := Bool(true) @@ -683,10 +683,10 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val co = conf.co implicit val ln = conf.ln val io = new Bundle { - val x_init = (new FIFOIO){(new LogicalNetworkIO) { new TransactionInit }}.flip - val x_init_data = (new FIFOIO){(new LogicalNetworkIO) { new TransactionInitData }}.flip - val p_rep = (new FIFOIO){(new LogicalNetworkIO) { new ProbeReply }}.flip - val p_rep_data = (new FIFOIO){(new LogicalNetworkIO) { new ProbeReplyData }}.flip + val x_init = (new FIFOIO){(new LogicalNetworkIO) { new Acquire }}.flip + val x_init_data = (new FIFOIO){(new LogicalNetworkIO) { new AcquireData }}.flip + val p_rep = (new FIFOIO){(new LogicalNetworkIO) { new Release }}.flip + val p_rep_data = (new FIFOIO){(new LogicalNetworkIO) { new ReleaseData }}.flip val free = Bool(INPUT) val tile_incoherent = Bits(INPUT, conf.ln.nTiles) val p_rep_data_dep = (new FIFOIO) { new TrackerDependency }.flip @@ -696,15 +696,15 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val mem_req_cmd = (new FIFOIO) { new MemReqCmd } val mem_req_data = (new FIFOIO) { new MemData } val mem_req_lock = Bool(OUTPUT) - val p_req = (new FIFOIO) {(new LogicalNetworkIO) { new ProbeRequest }} - val x_rep = (new FIFOIO) {(new LogicalNetworkIO) { new TransactionReply }} + val p_req = (new FIFOIO) {(new LogicalNetworkIO) { new Probe }} + val grant = (new FIFOIO) {(new LogicalNetworkIO) { new Grant }} val busy = Bool(OUTPUT) val has_conflict = Bool(OUTPUT) } val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) - val xact = Reg{ new TransactionInit } + val xact = Reg{ new Acquire } val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) //TODO: Will need id reg for merged release xacts val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) @@ -730,7 +730,7 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val myflag = Mux(probe_self, Bits(0), UFixToOH(io.x_init.bits.header.src(log2Up(conf.ln.nTiles)-1,0))) p_req_initial_flags := ~(io.tile_incoherent | myflag) } - val all_x_reps_require_acks = Bool(true) + val all_grants_require_acks = Bool(true) io.busy := state != s_idle io.has_conflict := co.isCoherenceConflict(xact.addr, io.x_init.bits.payload.addr) && (state != s_idle) @@ -742,24 +742,24 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com io.mem_req_data.bits.data := UFix(0) io.mem_req_lock := Bool(false) io.p_req.valid := Bool(false) - io.p_req.bits.payload.p_type := co.getProbeRequestType(xact.x_type, UFix(0)) - io.p_req.bits.payload.global_xact_id := UFix(id) + io.p_req.bits.payload.p_type := co.getProbeType(xact.a_type, UFix(0)) + io.p_req.bits.payload.master_xact_id := UFix(id) io.p_req.bits.payload.addr := xact.addr io.p_req.bits.header.dst := UFix(0) - io.x_rep.bits.payload.data := io.mem_resp.bits.data - io.x_rep.bits.payload.x_type := co.getTransactionReplyType(xact.x_type, init_sharer_cnt_) - io.x_rep.bits.payload.tile_xact_id := xact.tile_xact_id - io.x_rep.bits.payload.global_xact_id := UFix(id) - io.x_rep.bits.payload.require_ack := all_x_reps_require_acks - io.x_rep.bits.header.dst := init_client_id_ - io.x_rep.valid := (io.mem_resp.valid && (UFix(id) === io.mem_resp.bits.tag)) + io.grant.bits.payload.data := io.mem_resp.bits.data + io.grant.bits.payload.g_type := co.getGrantType(xact.a_type, init_sharer_cnt_) + io.grant.bits.payload.client_xact_id := xact.client_xact_id + io.grant.bits.payload.master_xact_id := UFix(id) + io.grant.bits.payload.require_ack := all_grants_require_acks + io.grant.bits.header.dst := init_client_id_ + io.grant.valid := (io.mem_resp.valid && (UFix(id) === io.mem_resp.bits.tag)) io.x_init.ready := Bool(false) io.x_init_data.ready := Bool(false) io.x_init_data_dep.ready := Bool(false) io.p_rep.ready := Bool(false) io.p_rep_data.ready := Bool(false) io.p_rep_data_dep.ready := Bool(false) - io.mem_resp.ready := io.x_rep.ready + io.mem_resp.ready := io.grant.ready switch (state) { is(s_idle) { @@ -768,7 +768,7 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com init_client_id_ := io.x_init.bits.header.src init_sharer_cnt_ := UFix(conf.ln.nTiles) // TODO: Broadcast only x_init_data_needs_write := co.messageHasData(io.x_init.bits.payload) - x_needs_read := co.needsMemRead(io.x_init.bits.payload.x_type, UFix(0)) + x_needs_read := co.needsMemRead(io.x_init.bits.payload.a_type, UFix(0)) p_req_flags := p_req_initial_flags mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) @@ -807,7 +807,7 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com p_rep_data_needs_write, p_w_mem_cmd_sent, io.p_rep_data_dep.ready, - io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id))) + io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.master_xact_id === UFix(id))) } . elsewhen(x_init_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, @@ -816,17 +816,17 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com x_init_data_needs_write, x_w_mem_cmd_sent, io.x_init_data_dep.ready, - io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id))) + io.x_init_data_dep.valid && (io.x_init_data_dep.bits.master_xact_id === UFix(id))) } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { - state := Mux(co.needsAckReply(xact.x_type, UFix(0)), s_ack, - Mux(all_x_reps_require_acks, s_busy, s_idle)) + state := Mux(co.needsAckReply(xact.a_type, UFix(0)), s_ack, + Mux(all_grants_require_acks, s_busy, s_idle)) } } is(s_ack) { - io.x_rep.valid := Bool(true) - when(io.x_rep.ready) { state := Mux(all_x_reps_require_acks, s_busy, s_idle) } + io.grant.valid := Bool(true) + when(io.grant.ready) { state := Mux(all_grants_require_acks, s_busy, s_idle) } } is(s_busy) { // Nothing left to do but wait for transaction to complete when (io.free) { From b5ccdab514aae4b642c5a88b6c6cbe767420133b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 22 Jan 2013 20:09:21 -0800 Subject: [PATCH 157/374] changed val names in hub to match new tilelink names --- uncore/src/uncore.scala | 462 ++++++++++++++++++++-------------------- 1 file changed, 231 insertions(+), 231 deletions(-) diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 690454a1..21aaba8c 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -24,14 +24,14 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val p_data = (new PipeIO) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) val grant_ack = Bool(INPUT) - val p_rep_cnt_dec = Bits(INPUT, conf.ln.nTiles) - val p_req_cnt_inc = Bits(INPUT, conf.ln.nTiles) + val release_cnt_dec = Bits(INPUT, conf.ln.nTiles) + val probe_cnt_inc = Bits(INPUT, conf.ln.nTiles) val tile_incoherent = Bits(INPUT, conf.ln.nTiles) - val p_rep_data = (new PipeIO) { new ReleaseData }.flip - val x_init_data = (new PipeIO) { new AcquireData }.flip + val release_data = (new PipeIO) { new ReleaseData }.flip + val acquire_data = (new PipeIO) { new AcquireData }.flip val sent_grant_ack = Bool(INPUT) - val p_rep_data_dep = (new PipeIO) { new TrackerDependency }.flip - val x_init_data_dep = (new PipeIO) { new TrackerDependency }.flip + val release_data_dep = (new PipeIO) { new TrackerDependency }.flip + val acquire_data_dep = (new PipeIO) { new TrackerDependency }.flip val mem_req_cmd = (new FIFOIO) { new MemReqCmd } val mem_req_data = (new FIFOIO) { new MemData } @@ -40,17 +40,17 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val busy = Bool(OUTPUT) val addr = Bits(OUTPUT, PADDR_BITS - OFFSET_BITS) val init_client_id = Bits(OUTPUT, conf.ln.idBits) - val p_rep_client_id = Bits(OUTPUT, conf.ln.idBits) + val release_client_id = Bits(OUTPUT, conf.ln.idBits) val client_xact_id = Bits(OUTPUT, CLIENT_XACT_ID_BITS) val sharer_count = Bits(OUTPUT, conf.ln.idBits+1) val a_type = Bits(OUTPUT, ACQUIRE_TYPE_MAX_BITS) - val push_p_req = Bits(OUTPUT, conf.ln.nTiles) - val pop_p_rep = Bits(OUTPUT, conf.ln.nTiles) - val pop_p_rep_data = Bits(OUTPUT, conf.ln.nTiles) - val pop_p_rep_dep = Bits(OUTPUT, conf.ln.nTiles) - val pop_x_init = Bits(OUTPUT, conf.ln.nTiles) - val pop_x_init_data = Bits(OUTPUT, conf.ln.nTiles) - val pop_x_init_dep = Bits(OUTPUT, conf.ln.nTiles) + val push_probe = Bits(OUTPUT, conf.ln.nTiles) + val pop_release = Bits(OUTPUT, conf.ln.nTiles) + val pop_release_data = Bits(OUTPUT, conf.ln.nTiles) + val pop_release_dep = Bits(OUTPUT, conf.ln.nTiles) + val pop_acquire = Bits(OUTPUT, conf.ln.nTiles) + val pop_acquire_data = Bits(OUTPUT, conf.ln.nTiles) + val pop_acquire_dep = Bits(OUTPUT, conf.ln.nTiles) val send_grant_ack = Bool(OUTPUT) } @@ -91,19 +91,19 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val state = Reg(resetVal = s_idle) val xact = Reg{ new Acquire } val init_client_id_ = Reg{ Bits() } - val p_rep_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) - val p_req_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) - val p_rep_client_id_ = Reg{ Bits() } + val release_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) + val release_client_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) - val x_init_data_needs_write = Reg(resetVal = Bool(false)) - val p_rep_data_needs_write = Reg(resetVal = Bool(false)) + val acquire_data_needs_write = Reg(resetVal = Bool(false)) + val release_data_needs_write = Reg(resetVal = Bool(false)) val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) - val p_req_initial_flags = Bits(width = conf.ln.nTiles) - p_req_initial_flags := Bits(0) + val probe_initial_flags = Bits(width = conf.ln.nTiles) + probe_initial_flags := Bits(0) if (conf.ln.nTiles > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence // TODO: this is hackish; figure out how to do it more systematically @@ -112,13 +112,13 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex case _ => Bool(false) } val myflag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.client_id(log2Up(conf.ln.nTiles)-1,0))) - p_req_initial_flags := ~(io.tile_incoherent | myflag) + probe_initial_flags := ~(io.tile_incoherent | myflag) } io.busy := state != s_idle io.addr := xact.addr io.init_client_id := init_client_id_ - io.p_rep_client_id := p_rep_client_id_ + io.release_client_id := release_client_id_ io.client_xact_id := xact.client_xact_id io.sharer_count := UFix(conf.ln.nTiles) // TODO: Broadcast only io.a_type := xact.a_type @@ -134,13 +134,13 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex io.probe.bits.p_type := co.getProbeType(xact.a_type, UFix(0)) io.probe.bits.master_xact_id := UFix(id) io.probe.bits.addr := xact.addr - io.push_p_req := Bits(0, width = conf.ln.nTiles) - io.pop_p_rep := Bits(0, width = conf.ln.nTiles) - io.pop_p_rep_data := Bits(0, width = conf.ln.nTiles) - io.pop_p_rep_dep := Bits(0, width = conf.ln.nTiles) - io.pop_x_init := Bits(0, width = conf.ln.nTiles) - io.pop_x_init_data := Bits(0, width = conf.ln.nTiles) - io.pop_x_init_dep := Bits(0, width = conf.ln.nTiles) + io.push_probe := Bits(0, width = conf.ln.nTiles) + io.pop_release := Bits(0, width = conf.ln.nTiles) + io.pop_release_data := Bits(0, width = conf.ln.nTiles) + io.pop_release_dep := Bits(0, width = conf.ln.nTiles) + io.pop_acquire := Bits(0, width = conf.ln.nTiles) + io.pop_acquire_data := Bits(0, width = conf.ln.nTiles) + io.pop_acquire_dep := Bits(0, width = conf.ln.nTiles) io.send_grant_ack := Bool(false) switch (state) { @@ -148,62 +148,62 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex when( io.alloc_req.valid && io.can_alloc ) { xact := io.alloc_req.bits.acquire init_client_id_ := io.alloc_req.bits.client_id - x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.acquire) + acquire_data_needs_write := co.messageHasData(io.alloc_req.bits.acquire) x_needs_read := co.needsMemRead(io.alloc_req.bits.acquire.a_type, UFix(0)) - p_req_flags := p_req_initial_flags + probe_flags := probe_initial_flags mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) - io.pop_x_init := UFix(1) << io.alloc_req.bits.client_id + io.pop_acquire := UFix(1) << io.alloc_req.bits.client_id if(conf.ln.nTiles > 1) { - p_rep_count := PopCount(p_req_initial_flags) - state := Mux(p_req_initial_flags.orR, s_probe, s_mem) + release_count := PopCount(probe_initial_flags) + state := Mux(probe_initial_flags.orR, s_probe, s_mem) } else state := s_mem } } is(s_probe) { - when(p_req_flags.orR) { - io.push_p_req := p_req_flags + when(probe_flags.orR) { + io.push_probe := probe_flags io.probe.valid := Bool(true) } - when(io.p_req_cnt_inc.orR) { - p_req_flags := p_req_flags & ~io.p_req_cnt_inc // unflag sent reqs + when(io.probe_cnt_inc.orR) { + probe_flags := probe_flags & ~io.probe_cnt_inc // unflag sent reqs } - when(io.p_rep_cnt_dec.orR) { - val dec = PopCount(io.p_rep_cnt_dec) - io.pop_p_rep := io.p_rep_cnt_dec - if(conf.ln.nTiles > 1) p_rep_count := p_rep_count - dec - when(p_rep_count === dec) { + when(io.release_cnt_dec.orR) { + val dec = PopCount(io.release_cnt_dec) + io.pop_release := io.release_cnt_dec + if(conf.ln.nTiles > 1) release_count := release_count - dec + when(release_count === dec) { state := s_mem } } when(io.p_data.valid) { - p_rep_data_needs_write := Bool(true) - p_rep_client_id_ := io.p_data.bits.client_id + release_data_needs_write := Bool(true) + release_client_id_ := io.p_data.bits.client_id } } is(s_mem) { - when (p_rep_data_needs_write) { + when (release_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, - io.p_rep_data, - p_rep_data_needs_write, + io.release_data, + release_data_needs_write, p_w_mem_cmd_sent, - io.pop_p_rep_data, - io.pop_p_rep_dep, - io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.master_xact_id === UFix(id)), - p_rep_client_id_) - } . elsewhen(x_init_data_needs_write) { + io.pop_release_data, + io.pop_release_dep, + io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(id)), + release_client_id_) + } . elsewhen(acquire_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, - io.x_init_data, - x_init_data_needs_write, + io.acquire_data, + acquire_data_needs_write, x_w_mem_cmd_sent, - io.pop_x_init_data, - io.pop_x_init_dep, - io.x_init_data_dep.valid && (io.x_init_data_dep.bits.master_xact_id === UFix(id)), + io.pop_acquire_data, + io.pop_acquire_dep, + io.acquire_data_dep.valid && (io.acquire_data_dep.bits.master_xact_id === UFix(id)), init_client_id_) } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) @@ -274,22 +274,22 @@ class CoherenceHubNull(implicit conf: CoherenceHubConfiguration) extends Coheren { val co = conf.co.asInstanceOf[ThreeStateIncoherence] - val x_init = io.tiles(0).acquire - val is_write = x_init.bits.payload.a_type === co.acquireWriteback - x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp - io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) + val acquire = io.tiles(0).acquire + val is_write = acquire.bits.payload.a_type === co.acquireWriteback + acquire.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp + io.mem.req_cmd.valid := acquire.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write - io.mem.req_cmd.bits.tag := x_init.bits.payload.client_xact_id - io.mem.req_cmd.bits.addr := x_init.bits.payload.addr + io.mem.req_cmd.bits.tag := acquire.bits.payload.client_xact_id + io.mem.req_cmd.bits.addr := acquire.bits.payload.addr io.mem.req_data <> io.tiles(0).acquire_data val grant = io.tiles(0).grant grant.bits.payload.g_type := Mux(io.mem.resp.valid, co.grantData, co.grantAck) - grant.bits.payload.client_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.payload.client_xact_id) + grant.bits.payload.client_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, acquire.bits.payload.client_xact_id) grant.bits.payload.master_xact_id := UFix(0) // don't care grant.bits.payload.data := io.mem.resp.bits.data grant.bits.payload.require_ack := Bool(true) - grant.valid := io.mem.resp.valid || x_init.valid && is_write && io.mem.req_cmd.ready + grant.valid := io.mem.resp.valid || acquire.valid && is_write && io.mem.req_cmd.ready io.tiles(0).abort.valid := Bool(false) io.tiles(0).grant_ack.ready := Bool(true) @@ -314,8 +314,8 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val send_grant_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } val do_free_arr = Vec(NGLOBAL_XACTS){ Bool() } - val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } - val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } + val release_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } + val probe_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } val sent_grant_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } val p_data_client_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } @@ -332,8 +332,8 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co t.grant_ack := do_free_arr(i) t.p_data.bits.client_id := p_data_client_id_arr(i) t.p_data.valid := p_data_valid_arr(i) - t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits - t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits + t.release_cnt_dec := release_cnt_dec_arr(i).toBits + t.probe_cnt_inc := probe_cnt_inc_arr(i).toBits t.tile_incoherent := io.incoherent.toBits t.sent_grant_ack := sent_grant_ack_arr(i) do_free_arr(i) := Bool(false) @@ -341,21 +341,21 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co p_data_client_id_arr(i) := Bits(0, width = conf.ln.idBits) p_data_valid_arr(i) := Bool(false) for( j <- 0 until conf.ln.nTiles) { - p_rep_cnt_dec_arr(i)(j) := Bool(false) - p_req_cnt_inc_arr(i)(j) := Bool(false) + release_cnt_dec_arr(i)(j) := Bool(false) + probe_cnt_inc_arr(i)(j) := Bool(false) } } - val p_rep_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY - val x_init_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY + val release_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY + val acquire_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY // Free finished transactions for( j <- 0 until conf.ln.nTiles ) { - val finish = io.tiles(j).grant_ack - when (finish.valid) { - do_free_arr(finish.bits.payload.master_xact_id) := Bool(true) + val ack = io.tiles(j).grant_ack + when (ack.valid) { + do_free_arr(ack.bits.payload.master_xact_id) := Bool(true) } - finish.ready := Bool(true) + ack.ready := Bool(true) } // Reply to initial requestor @@ -402,31 +402,31 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co // Handle probe replies, which may or may not have data for( j <- 0 until conf.ln.nTiles ) { - val p_rep = io.tiles(j).release - val p_rep_data = io.tiles(j).release_data - val idx = p_rep.bits.payload.master_xact_id - val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) - val do_pop = foldR(pop_p_reps)(_ || _) - p_rep.ready := Bool(true) - p_rep_data_dep_list(j).io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits.payload) - p_rep_data_dep_list(j).io.enq.bits.master_xact_id := p_rep.bits.payload.master_xact_id - p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) - when (p_rep.valid && co.messageHasData(p_rep.bits.payload)) { + val release = io.tiles(j).release + val release_data = io.tiles(j).release_data + val idx = release.bits.payload.master_xact_id + val pop_releases = trackerList.map(_.io.pop_release(j).toBool) + val do_pop = foldR(pop_releases)(_ || _) + release.ready := Bool(true) + release_data_dep_list(j).io.enq.valid := release.valid && co.messageHasData(release.bits.payload) + release_data_dep_list(j).io.enq.bits.master_xact_id := release.bits.payload.master_xact_id + release_data.ready := foldR(trackerList.map(_.io.pop_release_data(j)))(_ || _) + when (release.valid && co.messageHasData(release.bits.payload)) { p_data_valid_arr(idx) := Bool(true) p_data_client_id_arr(idx) := UFix(j) } - p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) + release_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_release_dep(j).toBool))(_||_) } for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_client_id).release_data.valid - trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_client_id).release_data.bits.payload + trackerList(i).io.release_data.valid := io.tiles(trackerList(i).io.release_client_id).release_data.valid + trackerList(i).io.release_data.bits := io.tiles(trackerList(i).io.release_client_id).release_data.bits.payload - trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_client_id, p_rep_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) - trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_client_id, p_rep_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) + trackerList(i).io.release_data_dep.valid := MuxLookup(trackerList(i).io.release_client_id, release_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> release_data_dep_list(j).io.deq.valid)) + trackerList(i).io.release_data_dep.bits := MuxLookup(trackerList(i).io.release_client_id, release_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> release_data_dep_list(j).io.deq.bits)) for( j <- 0 until conf.ln.nTiles) { - val p_rep = io.tiles(j).release - p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.payload.master_xact_id === UFix(i)) + val release = io.tiles(j).release + release_cnt_dec_arr(i)(j) := release.valid && (release.bits.payload.master_xact_id === UFix(i)) } } @@ -435,31 +435,31 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val abort_state_arr = Vec(conf.ln.nTiles) { Reg(resetVal = s_idle) } val want_to_abort_arr = Vec(conf.ln.nTiles) { Bool() } for( j <- 0 until conf.ln.nTiles ) { - val x_init = io.tiles(j).acquire - val x_init_data = io.tiles(j).acquire_data + val acquire = io.tiles(j).acquire + val acquire_data = io.tiles(j).acquire_data val x_abort = io.tiles(j).abort val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val conflicts = Vec(NGLOBAL_XACTS) { Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.payload.addr) + conflicts(i) := t.busy && acquire.valid && co.isCoherenceConflict(t.addr, acquire.bits.payload.addr) } - x_abort.bits.payload.client_xact_id := x_init.bits.payload.client_xact_id - want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits.payload))) + x_abort.bits.payload.client_xact_id := acquire.bits.payload.client_xact_id + want_to_abort_arr(j) := acquire.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!acquire_data_dep_list(j).io.enq.ready && co.messageHasData(acquire.bits.payload))) x_abort.valid := Bool(false) switch(abort_state_arr(j)) { is(s_idle) { when(want_to_abort_arr(j)) { - when(co.messageHasData(x_init.bits.payload)) { + when(co.messageHasData(acquire.bits.payload)) { abort_state_arr(j) := s_abort_drain } . otherwise { abort_state_arr(j) := s_abort_send } } } - is(s_abort_drain) { // raises x_init_data.ready below - when(x_init_data.valid) { + is(s_abort_drain) { // raises acquire_data.ready below + when(acquire_data.valid) { abort_cnt := abort_cnt + UFix(1) when(abort_cnt === ~UFix(0, width = log2Up(REFILL_CYCLES))) { abort_state_arr(j) := s_abort_send @@ -468,7 +468,7 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co } is(s_abort_send) { // nothing is dequeued for now x_abort.valid := Bool(true) - when(x_abort.ready) { // raises x_init.ready below + when(x_abort.ready) { // raises acquire.ready below abort_state_arr(j) := s_idle } } @@ -486,41 +486,41 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co trackerList(i).io.alloc_req.bits := init_arb.io.out.bits trackerList(i).io.alloc_req.valid := init_arb.io.out.valid - trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_client_id).acquire_data.bits.payload - trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_client_id).acquire_data.valid - trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_client_id, x_init_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) - trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_client_id, x_init_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) + trackerList(i).io.acquire_data.bits := io.tiles(trackerList(i).io.init_client_id).acquire_data.bits.payload + trackerList(i).io.acquire_data.valid := io.tiles(trackerList(i).io.init_client_id).acquire_data.valid + trackerList(i).io.acquire_data_dep.bits := MuxLookup(trackerList(i).io.init_client_id, acquire_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> acquire_data_dep_list(j).io.deq.bits)) + trackerList(i).io.acquire_data_dep.valid := MuxLookup(trackerList(i).io.init_client_id, acquire_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> acquire_data_dep_list(j).io.deq.valid)) } for( j <- 0 until conf.ln.nTiles ) { - val x_init = io.tiles(j).acquire - val x_init_data = io.tiles(j).acquire_data - val x_init_data_dep = x_init_data_dep_list(j).io.deq + val acquire = io.tiles(j).acquire + val acquire_data = io.tiles(j).acquire_data + val acquire_data_dep = acquire_data_dep_list(j).io.deq val x_abort = io.tiles(j).abort - init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid - init_arb.io.in(j).bits.acquire := x_init.bits.payload + init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && acquire.valid + init_arb.io.in(j).bits.acquire := acquire.bits.payload init_arb.io.in(j).bits.client_id := UFix(j) - val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) - val do_pop = foldR(pop_x_inits)(_||_) - x_init_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(x_init.bits.payload) && (abort_state_arr(j) === s_idle) - x_init_data_dep_list(j).io.enq.bits.master_xact_id := OHToUFix(pop_x_inits) - x_init.ready := (x_abort.valid && x_abort.ready) || do_pop - x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) - x_init_data_dep.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) + val pop_acquires = trackerList.map(_.io.pop_acquire(j).toBool) + val do_pop = foldR(pop_acquires)(_||_) + acquire_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(acquire.bits.payload) && (abort_state_arr(j) === s_idle) + acquire_data_dep_list(j).io.enq.bits.master_xact_id := OHToUFix(pop_acquires) + acquire.ready := (x_abort.valid && x_abort.ready) || do_pop + acquire_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_acquire_data(j).toBool))(_||_) + acquire_data_dep.ready := foldR(trackerList.map(_.io.pop_acquire_dep(j).toBool))(_||_) } alloc_arb.io.out.ready := init_arb.io.out.valid // Handle probe request generation // Must arbitrate for each request port - val p_req_arb_arr = List.fill(conf.ln.nTiles)((new Arbiter(NGLOBAL_XACTS)) { new Probe() }) + val probe_arb_arr = List.fill(conf.ln.nTiles)((new Arbiter(NGLOBAL_XACTS)) { new Probe() }) for( j <- 0 until conf.ln.nTiles ) { for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io - p_req_arb_arr(j).io.in(i).bits := t.probe.bits - p_req_arb_arr(j).io.in(i).valid := t.probe.valid && t.push_p_req(j) - p_req_cnt_inc_arr(i)(j) := p_req_arb_arr(j).io.in(i).ready + probe_arb_arr(j).io.in(i).bits := t.probe.bits + probe_arb_arr(j).io.in(i).valid := t.probe.valid && t.push_probe(j) + probe_cnt_inc_arr(i)(j) := probe_arb_arr(j).io.in(i).ready } - FIFOedLogicalNetworkIOWrapper(p_req_arb_arr(j).io.out) <> io.tiles(j).probe + FIFOedLogicalNetworkIOWrapper(probe_arb_arr(j).io.out) <> io.tiles(j).probe } } @@ -538,8 +538,8 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren implicit val lnConf = conf.ln val co = conf.co val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) - val p_rep_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth must >= NPRIMARY - val x_init_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth should >= NPRIMARY + val release_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth must >= NPRIMARY + val acquire_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth should >= NPRIMARY for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i) @@ -552,49 +552,49 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren // Handle transaction initiation requests // Only one allocation per cycle // Init requests may or may not have data - val x_init = io.network.acquire - val x_init_data = io.network.acquire_data + val acquire = io.network.acquire + val acquire_data = io.network.acquire_data val x_abort = io.network.abort - val x_dep_deq = x_init_data_dep_q.io.deq + val x_dep_deq = acquire_data_dep_q.io.deq val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } val abort_state = Reg(resetVal = s_idle) val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val any_conflict = trackerList.map(_.io.has_conflict).reduce(_||_) val all_busy = trackerList.map(_.io.busy).reduce(_&&_) - val want_to_abort = x_init.valid && (any_conflict || all_busy || (!x_init_data_dep_q.io.enq.ready && co.messageHasData(x_init.bits.payload))) + val want_to_abort = acquire.valid && (any_conflict || all_busy || (!acquire_data_dep_q.io.enq.ready && co.messageHasData(acquire.bits.payload))) val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } for( i <- 0 until NGLOBAL_XACTS ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy - trackerList(i).io.x_init.bits := x_init.bits - trackerList(i).io.x_init.valid := (abort_state === s_idle) && !want_to_abort && x_init.valid && alloc_arb.io.in(i).ready + trackerList(i).io.acquire.bits := acquire.bits + trackerList(i).io.acquire.valid := (abort_state === s_idle) && !want_to_abort && acquire.valid && alloc_arb.io.in(i).ready - trackerList(i).io.x_init_data.bits := x_init_data.bits - trackerList(i).io.x_init_data.valid := x_init_data.valid - trackerList(i).io.x_init_data_dep.bits := x_dep_deq.bits - trackerList(i).io.x_init_data_dep.valid := x_dep_deq.valid + trackerList(i).io.acquire_data.bits := acquire_data.bits + trackerList(i).io.acquire_data.valid := acquire_data.valid + trackerList(i).io.acquire_data_dep.bits := x_dep_deq.bits + trackerList(i).io.acquire_data_dep.valid := x_dep_deq.valid } - val pop_x_init = trackerList.map(_.io.x_init.ready).reduce(_||_) - x_init.ready := (x_abort.valid && x_abort.ready) || pop_x_init - x_init_data.ready := (abort_state === s_abort_drain) || trackerList.map(_.io.x_init_data.ready).reduce(_||_) - x_init_data_dep_q.io.enq.valid := pop_x_init && co.messageHasData(x_init.bits.payload) && (abort_state === s_idle) - x_init_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(trackerList.map(_.io.x_init.ready)) - x_dep_deq.ready := trackerList.map(_.io.x_init_data_dep.ready).reduce(_||_) + val pop_acquire = trackerList.map(_.io.acquire.ready).reduce(_||_) + acquire.ready := (x_abort.valid && x_abort.ready) || pop_acquire + acquire_data.ready := (abort_state === s_abort_drain) || trackerList.map(_.io.acquire_data.ready).reduce(_||_) + acquire_data_dep_q.io.enq.valid := pop_acquire && co.messageHasData(acquire.bits.payload) && (abort_state === s_idle) + acquire_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(trackerList.map(_.io.acquire.ready)) + x_dep_deq.ready := trackerList.map(_.io.acquire_data_dep.ready).reduce(_||_) - alloc_arb.io.out.ready := x_init.valid + alloc_arb.io.out.ready := acquire.valid // Nack conflicting transaction init attempts - x_abort.bits.header.dst := x_init.bits.header.src - x_abort.bits.payload.client_xact_id := x_init.bits.payload.client_xact_id + x_abort.bits.header.dst := acquire.bits.header.src + x_abort.bits.payload.client_xact_id := acquire.bits.payload.client_xact_id x_abort.valid := Bool(false) switch(abort_state) { is(s_idle) { when(want_to_abort) { - abort_state := Mux( co.messageHasData(x_init.bits.payload), s_abort_drain, s_abort_send) + abort_state := Mux( co.messageHasData(acquire.bits.payload), s_abort_drain, s_abort_send) } } - is(s_abort_drain) { // raises x_init_data.ready below - when(x_init_data.valid) { + is(s_abort_drain) { // raises acquire_data.ready below + when(acquire_data.valid) { abort_cnt := abort_cnt + UFix(1) when(abort_cnt === ~UFix(0, width = log2Up(REFILL_CYCLES))) { abort_state := s_abort_send @@ -603,38 +603,38 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren } is(s_abort_send) { // nothing is dequeued for now x_abort.valid := Bool(true) - when(x_abort.ready) { // raises x_init.ready + when(x_abort.ready) { // raises acquire.ready abort_state := s_idle } } } // Handle probe request generation - val p_req_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new Probe }} + val probe_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new Probe }} for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io - p_req_arb.io.in(i).bits := t.p_req.bits - p_req_arb.io.in(i).valid := t.p_req.valid - t.p_req.ready := p_req_arb.io.in(i).ready + probe_arb.io.in(i).bits := t.probe.bits + probe_arb.io.in(i).valid := t.probe.valid + t.probe.ready := probe_arb.io.in(i).ready } - io.network.probe <> p_req_arb.io.out + io.network.probe <> probe_arb.io.out // Handle probe replies, which may or may not have data - val p_rep = io.network.release - val p_rep_data = io.network.release_data - val idx = p_rep.bits.payload.master_xact_id - p_rep.ready := trackerList.map(_.io.p_rep.ready).reduce(_||_) - p_rep_data.ready := trackerList.map(_.io.p_rep_data.ready).reduce(_||_) - p_rep_data_dep_q.io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits.payload) - p_rep_data_dep_q.io.enq.bits.master_xact_id := p_rep.bits.payload.master_xact_id - p_rep_data_dep_q.io.deq.ready := trackerList.map(_.io.p_rep_data_dep.ready).reduce(_||_) + val release = io.network.release + val release_data = io.network.release_data + val idx = release.bits.payload.master_xact_id + release.ready := trackerList.map(_.io.release.ready).reduce(_||_) + release_data.ready := trackerList.map(_.io.release_data.ready).reduce(_||_) + release_data_dep_q.io.enq.valid := release.valid && co.messageHasData(release.bits.payload) + release_data_dep_q.io.enq.bits.master_xact_id := release.bits.payload.master_xact_id + release_data_dep_q.io.deq.ready := trackerList.map(_.io.release_data_dep.ready).reduce(_||_) for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.p_rep_data.valid := p_rep_data.valid - trackerList(i).io.p_rep_data.bits := p_rep_data.bits - trackerList(i).io.p_rep_data_dep.valid := p_rep_data_dep_q.io.deq.valid - trackerList(i).io.p_rep_data_dep.bits := p_rep_data_dep_q.io.deq.bits - trackerList(i).io.p_rep.valid := p_rep.valid && (idx === UFix(i)) - trackerList(i).io.p_rep.bits := p_rep.bits + trackerList(i).io.release_data.valid := release_data.valid + trackerList(i).io.release_data.bits := release_data.bits + trackerList(i).io.release_data_dep.valid := release_data_dep_q.io.deq.valid + trackerList(i).io.release_data_dep.bits := release_data_dep_q.io.deq.bits + trackerList(i).io.release.valid := release.valid && (idx === UFix(i)) + trackerList(i).io.release.bits := release.bits } // Reply to initial requestor @@ -659,11 +659,11 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren } // Free finished transactions - val finish = io.network.grant_ack + val ack = io.network.grant_ack for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.free := finish.valid && (finish.bits.payload.master_xact_id === UFix(i)) + trackerList(i).io.free := ack.valid && (ack.bits.payload.master_xact_id === UFix(i)) } - finish.ready := Bool(true) + ack.ready := Bool(true) // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests @@ -683,20 +683,20 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val co = conf.co implicit val ln = conf.ln val io = new Bundle { - val x_init = (new FIFOIO){(new LogicalNetworkIO) { new Acquire }}.flip - val x_init_data = (new FIFOIO){(new LogicalNetworkIO) { new AcquireData }}.flip - val p_rep = (new FIFOIO){(new LogicalNetworkIO) { new Release }}.flip - val p_rep_data = (new FIFOIO){(new LogicalNetworkIO) { new ReleaseData }}.flip + val acquire = (new FIFOIO){(new LogicalNetworkIO) { new Acquire }}.flip + val acquire_data = (new FIFOIO){(new LogicalNetworkIO) { new AcquireData }}.flip + val release = (new FIFOIO){(new LogicalNetworkIO) { new Release }}.flip + val release_data = (new FIFOIO){(new LogicalNetworkIO) { new ReleaseData }}.flip val free = Bool(INPUT) val tile_incoherent = Bits(INPUT, conf.ln.nTiles) - val p_rep_data_dep = (new FIFOIO) { new TrackerDependency }.flip - val x_init_data_dep = (new FIFOIO) { new TrackerDependency }.flip + val release_data_dep = (new FIFOIO) { new TrackerDependency }.flip + val acquire_data_dep = (new FIFOIO) { new TrackerDependency }.flip val mem_resp = (new FIFOIO) { new MemResp }.flip val mem_req_cmd = (new FIFOIO) { new MemReqCmd } val mem_req_data = (new FIFOIO) { new MemData } val mem_req_lock = Bool(OUTPUT) - val p_req = (new FIFOIO) {(new LogicalNetworkIO) { new Probe }} + val probe = (new FIFOIO) {(new LogicalNetworkIO) { new Probe }} val grant = (new FIFOIO) {(new LogicalNetworkIO) { new Grant }} val busy = Bool(OUTPUT) val has_conflict = Bool(OUTPUT) @@ -708,32 +708,32 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) //TODO: Will need id reg for merged release xacts val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) - val p_rep_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) - val p_req_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) + val release_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) val x_needs_read = Reg(resetVal = Bool(false)) - val x_init_data_needs_write = Reg(resetVal = Bool(false)) - val p_rep_data_needs_write = Reg(resetVal = Bool(false)) + val acquire_data_needs_write = Reg(resetVal = Bool(false)) + val release_data_needs_write = Reg(resetVal = Bool(false)) val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) - val p_req_initial_flags = Bits(width = conf.ln.nTiles) - p_req_initial_flags := Bits(0) + val probe_initial_flags = Bits(width = conf.ln.nTiles) + probe_initial_flags := Bits(0) if (conf.ln.nTiles > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence // TODO: this is hackish; figure out how to do it more systematically val probe_self = co match { - case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.x_init.bits.payload) + case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.acquire.bits.payload) case _ => Bool(false) } - val myflag = Mux(probe_self, Bits(0), UFixToOH(io.x_init.bits.header.src(log2Up(conf.ln.nTiles)-1,0))) - p_req_initial_flags := ~(io.tile_incoherent | myflag) + val myflag = Mux(probe_self, Bits(0), UFixToOH(io.acquire.bits.header.src(log2Up(conf.ln.nTiles)-1,0))) + probe_initial_flags := ~(io.tile_incoherent | myflag) } val all_grants_require_acks = Bool(true) io.busy := state != s_idle - io.has_conflict := co.isCoherenceConflict(xact.addr, io.x_init.bits.payload.addr) && (state != s_idle) + io.has_conflict := co.isCoherenceConflict(xact.addr, io.acquire.bits.payload.addr) && (state != s_idle) io.mem_req_cmd.valid := Bool(false) io.mem_req_cmd.bits.rw := Bool(false) io.mem_req_cmd.bits.addr := xact.addr @@ -741,11 +741,11 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com io.mem_req_data.valid := Bool(false) io.mem_req_data.bits.data := UFix(0) io.mem_req_lock := Bool(false) - io.p_req.valid := Bool(false) - io.p_req.bits.payload.p_type := co.getProbeType(xact.a_type, UFix(0)) - io.p_req.bits.payload.master_xact_id := UFix(id) - io.p_req.bits.payload.addr := xact.addr - io.p_req.bits.header.dst := UFix(0) + io.probe.valid := Bool(false) + io.probe.bits.payload.p_type := co.getProbeType(xact.a_type, UFix(0)) + io.probe.bits.payload.master_xact_id := UFix(id) + io.probe.bits.payload.addr := xact.addr + io.probe.bits.header.dst := UFix(0) io.grant.bits.payload.data := io.mem_resp.bits.data io.grant.bits.payload.g_type := co.getGrantType(xact.a_type, init_sharer_cnt_) io.grant.bits.payload.client_xact_id := xact.client_xact_id @@ -753,70 +753,70 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com io.grant.bits.payload.require_ack := all_grants_require_acks io.grant.bits.header.dst := init_client_id_ io.grant.valid := (io.mem_resp.valid && (UFix(id) === io.mem_resp.bits.tag)) - io.x_init.ready := Bool(false) - io.x_init_data.ready := Bool(false) - io.x_init_data_dep.ready := Bool(false) - io.p_rep.ready := Bool(false) - io.p_rep_data.ready := Bool(false) - io.p_rep_data_dep.ready := Bool(false) + io.acquire.ready := Bool(false) + io.acquire_data.ready := Bool(false) + io.acquire_data_dep.ready := Bool(false) + io.release.ready := Bool(false) + io.release_data.ready := Bool(false) + io.release_data_dep.ready := Bool(false) io.mem_resp.ready := io.grant.ready switch (state) { is(s_idle) { - when( io.x_init.valid ) { - xact := io.x_init.bits.payload - init_client_id_ := io.x_init.bits.header.src + when( io.acquire.valid ) { + xact := io.acquire.bits.payload + init_client_id_ := io.acquire.bits.header.src init_sharer_cnt_ := UFix(conf.ln.nTiles) // TODO: Broadcast only - x_init_data_needs_write := co.messageHasData(io.x_init.bits.payload) - x_needs_read := co.needsMemRead(io.x_init.bits.payload.a_type, UFix(0)) - p_req_flags := p_req_initial_flags + acquire_data_needs_write := co.messageHasData(io.acquire.bits.payload) + x_needs_read := co.needsMemRead(io.acquire.bits.payload.a_type, UFix(0)) + probe_flags := probe_initial_flags mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) - io.x_init.ready := Bool(true) + io.acquire.ready := Bool(true) if(conf.ln.nTiles > 1) { - p_rep_count := PopCount(p_req_initial_flags) - state := Mux(p_req_initial_flags.orR, s_probe, s_mem) + release_count := PopCount(probe_initial_flags) + state := Mux(probe_initial_flags.orR, s_probe, s_mem) } else state := s_mem } } is(s_probe) { - val curr_p_id = PriorityEncoder(p_req_flags) - when(p_req_flags.orR) { - io.p_req.valid := Bool(true) - io.p_req.bits.header.dst := curr_p_id + val curr_p_id = PriorityEncoder(probe_flags) + when(probe_flags.orR) { + io.probe.valid := Bool(true) + io.probe.bits.header.dst := curr_p_id } - when(io.p_req.ready) { - p_req_flags := p_req_flags & ~(UFixToOH(curr_p_id)) + when(io.probe.ready) { + probe_flags := probe_flags & ~(UFixToOH(curr_p_id)) } - when(io.p_rep.valid) { - io.p_rep.ready := Bool(true) - if(conf.ln.nTiles > 1) p_rep_count := p_rep_count - UFix(1) - when(p_rep_count === UFix(1)) { + when(io.release.valid) { + io.release.ready := Bool(true) + if(conf.ln.nTiles > 1) release_count := release_count - UFix(1) + when(release_count === UFix(1)) { state := s_mem } - p_rep_data_needs_write := co.messageHasData(io.p_rep.bits.payload) + release_data_needs_write := co.messageHasData(io.release.bits.payload) } } is(s_mem) { - when (p_rep_data_needs_write) { + when (release_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, - io.p_rep_data, - p_rep_data_needs_write, + io.release_data, + release_data_needs_write, p_w_mem_cmd_sent, - io.p_rep_data_dep.ready, - io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.master_xact_id === UFix(id))) - } . elsewhen(x_init_data_needs_write) { + io.release_data_dep.ready, + io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(id))) + } . elsewhen(acquire_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, - io.x_init_data, - x_init_data_needs_write, + io.acquire_data, + acquire_data_needs_write, x_w_mem_cmd_sent, - io.x_init_data_dep.ready, - io.x_init_data_dep.valid && (io.x_init_data_dep.bits.master_xact_id === UFix(id))) + io.acquire_data_dep.ready, + io.acquire_data_dep.valid && (io.acquire_data_dep.bits.master_xact_id === UFix(id))) } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { From 1945fa898bb2ab1e242f201112a352eaf4f55dbe Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 24 Jan 2013 23:40:47 -0800 Subject: [PATCH 158/374] make external clock divider programmable --- uncore/src/slowio.scala | 47 ++++++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/uncore/src/slowio.scala b/uncore/src/slowio.scala index fa6fef94..cf00ae6e 100644 --- a/uncore/src/slowio.scala +++ b/uncore/src/slowio.scala @@ -3,7 +3,7 @@ package uncore import Chisel._ import Constants._ -class slowIO[T <: Data](val divisor: Int, hold_cycles_in: Int = -1)(data: => T) extends Component +class SlowIO[T <: Data](val divisor_max: Int)(data: => T) extends Component { val io = new Bundle { val out_fast = new FIFOIO()(data).flip @@ -13,31 +13,54 @@ class slowIO[T <: Data](val divisor: Int, hold_cycles_in: Int = -1)(data: => T) val in_slow = new FIFOIO()(data).flip val clk_slow = Bool(OUTPUT) + + val set_divisor = new PipeIO()(Bits(width = 32)).flip + val divisor = Bits(OUTPUT, 32) } - val hold_cycles = if (hold_cycles_in == -1) divisor/4 else hold_cycles_in - require((divisor & (divisor-1)) == 0) - require(hold_cycles < divisor/2 && hold_cycles >= 1) + require(divisor_max >= 8 && divisor_max <= 65536 && isPow2(divisor_max)) + val divisor = Reg(resetVal = UFix(divisor_max-1)) + val d_shadow = Reg(resetVal = UFix(divisor_max-1)) + val hold = Reg(resetVal = UFix(divisor_max/4-1)) + val h_shadow = Reg(resetVal = UFix(divisor_max/4-1)) + when (io.set_divisor.valid) { + d_shadow := io.set_divisor.bits(log2Up(divisor_max)-1, 0).toUFix + h_shadow := io.set_divisor.bits(log2Up(divisor_max)-1+16, 16).toUFix + } + io.divisor := hold << UFix(16) | divisor - val cnt = Reg() { UFix(width = log2Up(divisor)) } - cnt := cnt + UFix(1) - val out_en = cnt === UFix(divisor/2+hold_cycles-1) // rising edge + hold time - val in_en = cnt === UFix(divisor/2-1) // rising edge + val count = Reg{UFix(width = log2Up(divisor_max))} + val clock = Reg{Bool()} + count := count + UFix(1) + + val rising = count === (divisor >> UFix(1)) + val falling = count === divisor + val held = count === (divisor >> UFix(1)) + hold + + when (falling) { + divisor := d_shadow + hold := h_shadow + count := UFix(0) + clock := Bool(false) + } + when (rising) { + clock := Bool(true) + } val in_slow_rdy = Reg(resetVal = Bool(false)) val out_slow_val = Reg(resetVal = Bool(false)) val out_slow_bits = Reg() { data } val fromhost_q = new Queue(1)(data) - fromhost_q.io.enq.valid := in_en && (io.in_slow.valid && in_slow_rdy || reset) + fromhost_q.io.enq.valid := rising && (io.in_slow.valid && in_slow_rdy || reset) fromhost_q.io.enq.bits := io.in_slow.bits fromhost_q.io.deq <> io.in_fast val tohost_q = new Queue(1)(data) tohost_q.io.enq <> io.out_fast - tohost_q.io.deq.ready := in_en && io.out_slow.ready && out_slow_val + tohost_q.io.deq.ready := rising && io.out_slow.ready && out_slow_val - when (out_en) { + when (held) { in_slow_rdy := fromhost_q.io.enq.ready out_slow_val := tohost_q.io.deq.valid out_slow_bits := Mux(reset, fromhost_q.io.deq.bits, tohost_q.io.deq.bits) @@ -46,5 +69,5 @@ class slowIO[T <: Data](val divisor: Int, hold_cycles_in: Int = -1)(data: => T) io.in_slow.ready := in_slow_rdy io.out_slow.valid := out_slow_val io.out_slow.bits := out_slow_bits - io.clk_slow := cnt(log2Up(divisor)-1).toBool + io.clk_slow := clock } From 1134bbf1a4dbc9f479fab240410be8ac0ed1c514 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 27 Jan 2013 11:59:17 -0800 Subject: [PATCH 159/374] cleanup disconnected io pins (overwritten network headers) --- uncore/src/package.scala | 8 +++++--- uncore/src/uncore.scala | 7 +++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 0e69ff47..46d46661 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -62,21 +62,23 @@ class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { } object FIFOedLogicalNetworkIOWrapper { - def apply[T <: Data](in: FIFOIO[T])(implicit conf: LogicalNetworkConfiguration) = { - val shim = (new FIFOedLogicalNetworkIOWrapper){ in.bits.clone } + def apply[T <: Data](in: FIFOIO[T], src: UFix = UFix(0), dst: UFix = UFix(0))(implicit conf: LogicalNetworkConfiguration) = { + val shim = (new FIFOedLogicalNetworkIOWrapper(src, dst)){ in.bits.clone } shim.io.in.valid := in.valid shim.io.in.bits := in.bits in.ready := shim.io.in.ready shim.io.out } } -class FIFOedLogicalNetworkIOWrapper[T <: Data]()(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { +class FIFOedLogicalNetworkIOWrapper[T <: Data](src: UFix, dst: UFix)(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { val io = new Bundle { val in = (new FIFOIO){ data }.flip val out = (new FIFOIO){(new LogicalNetworkIO){ data }} } io.out.valid := io.in.valid io.out.bits.payload := io.in.bits + io.out.bits.header.dst := dst + io.out.bits.header.src := src io.in.ready := io.out.ready } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 690454a1..01e9c40c 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -369,6 +369,8 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co rep.bits.payload.master_xact_id := UFix(0) rep.bits.payload.data := io.mem.resp.bits.data rep.bits.payload.require_ack := Bool(true) + rep.bits.header.dst := UFix(0) // DNC + rep.bits.header.src := UFix(0) // DNC rep.valid := Bool(false) when(io.mem.resp.valid && (UFix(j) === init_client_id_arr(mem_idx))) { rep.bits.payload.g_type := co.getGrantType(a_type_arr(mem_idx), sh_count_arr(mem_idx)) @@ -445,6 +447,8 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.payload.addr) } x_abort.bits.payload.client_xact_id := x_init.bits.payload.client_xact_id + x_abort.bits.header.dst := UFix(0) // DNC + x_abort.bits.header.src := UFix(0) // DNC want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits.payload))) x_abort.valid := Bool(false) @@ -585,6 +589,7 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren // Nack conflicting transaction init attempts x_abort.bits.header.dst := x_init.bits.header.src + x_abort.bits.header.src := UFix(0) //DNC x_abort.bits.payload.client_xact_id := x_init.bits.payload.client_xact_id x_abort.valid := Bool(false) switch(abort_state) { @@ -746,12 +751,14 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com io.p_req.bits.payload.master_xact_id := UFix(id) io.p_req.bits.payload.addr := xact.addr io.p_req.bits.header.dst := UFix(0) + io.p_req.bits.header.src := UFix(0) // DNC io.grant.bits.payload.data := io.mem_resp.bits.data io.grant.bits.payload.g_type := co.getGrantType(xact.a_type, init_sharer_cnt_) io.grant.bits.payload.client_xact_id := xact.client_xact_id io.grant.bits.payload.master_xact_id := UFix(id) io.grant.bits.payload.require_ack := all_grants_require_acks io.grant.bits.header.dst := init_client_id_ + io.grant.bits.header.src := UFix(0) // DNC io.grant.valid := (io.mem_resp.valid && (UFix(id) === io.mem_resp.bits.tag)) io.x_init.ready := Bool(false) io.x_init_data.ready := Bool(false) From 47a632cc59cb469458e1a6a8082ea4a0df64d403 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 28 Jan 2013 16:39:45 -0800 Subject: [PATCH 160/374] added support for voluntary wbs over the release network --- uncore/src/coherence.scala | 84 ++++++++++++++++++++--- uncore/src/package.scala | 15 ----- uncore/src/tilelink.scala | 15 ++++- uncore/src/uncore.scala | 135 ++++++++++++++++++++++++++++--------- 4 files changed, 189 insertions(+), 60 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index 4268fcf8..e1cc453e 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -31,6 +31,7 @@ abstract class CoherencePolicy { def getAcquireTypeOnCacheControl(cmd: Bits): Bits def getAcquireTypeOnWriteback(): Bits + def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix): Release def newRelease (incoming: Probe, state: UFix): Release def messageHasData (reply: Release): Bool @@ -40,11 +41,15 @@ abstract class CoherencePolicy { def messageIsUncached(acq: Acquire): Bool def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool + def isVoluntary(rel: Release): Bool def getGrantType(a_type: UFix, count: UFix): Bits + def getGrantType(rel: Release, count: UFix): Bits def getProbeType(a_type: UFix, global_state: UFix): UFix def needsMemRead(a_type: UFix, global_state: UFix): Bool def needsMemWrite(a_type: UFix, global_state: UFix): Bool def needsAckReply(a_type: UFix, global_state: UFix): Bool + def requiresAck(grant: Grant): Bool + def requiresAck(release: Release): Bool } trait UncachedTransactions { @@ -70,17 +75,20 @@ abstract class IncoherentPolicy extends CoherencePolicy { def messageHasData (reply: Release) = Bool(false) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = Bool(false) def getGrantType(a_type: UFix, count: UFix): Bits = Bits(0) + def getGrantType(rel: Release, count: UFix): Bits = Bits(0) def getProbeType(a_type: UFix, global_state: UFix): UFix = UFix(0) def needsMemRead(a_type: UFix, global_state: UFix): Bool = Bool(false) def needsMemWrite(a_type: UFix, global_state: UFix): Bool = Bool(false) def needsAckReply(a_type: UFix, global_state: UFix): Bool = Bool(false) + def requiresAck(grant: Grant) = Bool(true) + def requiresAck(release: Release) = Bool(false) } class ThreeStateIncoherence extends IncoherentPolicy { val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(3){ UFix() } val grantData :: grantAck :: Nil = Enum(2){ UFix() } - val releaseInvalidateAck :: Nil = Enum(1){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateAck :: Nil = Enum(2){ UFix() } val uncachedTypeList = List() val hasDataTypeList = List(acquireWriteback) @@ -106,6 +114,9 @@ class ThreeStateIncoherence extends IncoherentPolicy { )) } + def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData + def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) Mux(write || cmd === M_PFW, acquireReadDirty, acquireReadClean) @@ -129,9 +140,9 @@ class MICoherence extends CoherencePolicyWithUncached { val globalInvalid :: globalValid :: Nil = Enum(2){ UFix() } val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(6){ UFix() } - val grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(6){ UFix() } + val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(7){ UFix() } val probeInvalidate :: probeCopy :: Nil = Enum(2){ UFix() } - val releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(4){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(5){ UFix() } val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, grantReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -180,7 +191,9 @@ class MICoherence extends CoherencePolicyWithUncached { def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) + def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = acquireReadExclusive def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = acquireReadExclusive @@ -228,6 +241,12 @@ class MICoherence extends CoherencePolicyWithUncached { )) } + def getGrantType(rel: Release, count: UFix): Bits = { + MuxLookup(rel.r_type, grantReadUncached, Array( + releaseVoluntaryInvalidateData -> grantVoluntaryAck + )) + } + def getProbeType(a_type: UFix, global_state: UFix): UFix = { MuxLookup(a_type, probeCopy, Array( acquireReadExclusive -> probeInvalidate, @@ -248,6 +267,8 @@ class MICoherence extends CoherencePolicyWithUncached { def needsAckReply(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } + def requiresAck(grant: Grant) = Bool(true) + def requiresAck(release: Release) = Bool(false) } class MEICoherence extends CoherencePolicyWithUncached { @@ -256,9 +277,9 @@ class MEICoherence extends CoherencePolicyWithUncached { val globalInvalid :: globalExclusiveClean :: Nil = Enum(2){ UFix() } val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(7){ UFix() } - val grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(7){ UFix() } + val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(8){ UFix() } val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } - val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(6){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(7){ UFix() } val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, grantReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -316,7 +337,9 @@ class MEICoherence extends CoherencePolicyWithUncached { def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) + def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -373,6 +396,12 @@ class MEICoherence extends CoherencePolicyWithUncached { acquireAtomicUncached -> grantAtomicUncached )) } + def getGrantType(rel: Release, count: UFix): Bits = { + MuxLookup(rel.r_type, grantReadUncached, Array( + releaseVoluntaryInvalidateData -> grantVoluntaryAck + )) + } + def getProbeType(a_type: UFix, global_state: UFix): UFix = { MuxLookup(a_type, probeCopy, Array( @@ -395,6 +424,8 @@ class MEICoherence extends CoherencePolicyWithUncached { def needsAckReply(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } + def requiresAck(grant: Grant) = Bool(true) + def requiresAck(release: Release) = Bool(false) } class MSICoherence extends CoherencePolicyWithUncached { @@ -403,9 +434,9 @@ class MSICoherence extends CoherencePolicyWithUncached { val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(3){ UFix() } val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(7){ UFix() } - val grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(8){ UFix() } + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(9){ UFix() } val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } - val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(6){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(7){ UFix() } val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, grantReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -470,7 +501,9 @@ class MSICoherence extends CoherencePolicyWithUncached { def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) + def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -527,6 +560,12 @@ class MSICoherence extends CoherencePolicyWithUncached { acquireAtomicUncached -> grantAtomicUncached )) } + def getGrantType(rel: Release, count: UFix): Bits = { + MuxLookup(rel.r_type, grantReadUncached, Array( + releaseVoluntaryInvalidateData -> grantVoluntaryAck + )) + } + def getProbeType(a_type: UFix, global_state: UFix): UFix = { MuxLookup(a_type, probeCopy, Array( @@ -546,6 +585,8 @@ class MSICoherence extends CoherencePolicyWithUncached { def needsAckReply(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } + def requiresAck(grant: Grant) = Bool(true) + def requiresAck(release: Release) = Bool(false) } class MESICoherence extends CoherencePolicyWithUncached { @@ -554,9 +595,9 @@ class MESICoherence extends CoherencePolicyWithUncached { val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(3){ UFix() } val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(7){ UFix() } - val grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(8){ UFix() } + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(9){ UFix() } val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } - val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(6){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(7){ UFix() } val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -621,7 +662,9 @@ class MESICoherence extends CoherencePolicyWithUncached { def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) + def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -678,6 +721,12 @@ class MESICoherence extends CoherencePolicyWithUncached { acquireAtomicUncached -> grantAtomicUncached )) } + def getGrantType(rel: Release, count: UFix): Bits = { + MuxLookup(rel.r_type, grantReadUncached, Array( + releaseVoluntaryInvalidateData -> grantVoluntaryAck + )) + } + def getProbeType(a_type: UFix, global_state: UFix): UFix = { MuxLookup(a_type, probeCopy, Array( @@ -700,6 +749,9 @@ class MESICoherence extends CoherencePolicyWithUncached { def needsAckReply(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } + + def requiresAck(grant: Grant) = Bool(true) + def requiresAck(release: Release) = Bool(false) } class MigratoryCoherence extends CoherencePolicyWithUncached { @@ -707,9 +759,9 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: tileSharedByTwo :: tileMigratoryClean :: tileMigratoryDirty :: Nil = Enum(7){ UFix() } val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(8){ UFix() } - val grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(9){ UFix() } + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(10){ UFix() } val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(4){ UFix() } - val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(10){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(11){ UFix() } val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -789,7 +841,9 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) + def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -848,6 +902,12 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI? )) } + def getGrantType(rel: Release, count: UFix): Bits = { + MuxLookup(rel.r_type, grantReadUncached, Array( + releaseVoluntaryInvalidateData -> grantVoluntaryAck + )) + } + def getProbeType(a_type: UFix, global_state: UFix): UFix = { MuxLookup(a_type, probeCopy, Array( @@ -871,4 +931,6 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def needsAckReply(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached ||a_type === acquireInvalidateOthers) } + def requiresAck(grant: Grant) = Bool(true) + def requiresAck(release: Release) = Bool(false) } diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 0e69ff47..8e511663 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -105,19 +105,4 @@ class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkCon override def clone = { new LogicalNetworkIO()(data).asInstanceOf[this.type] } } -abstract class DirectionalFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) -class ClientSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) -class MasterSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) {flip()} - -class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val acquire = (new ClientSourcedIO){(new LogicalNetworkIO){new Acquire }} - val acquire_data = (new ClientSourcedIO){(new LogicalNetworkIO){new AcquireData }} - val abort = (new MasterSourcedIO){(new LogicalNetworkIO){new Abort }} - val probe = (new MasterSourcedIO){(new LogicalNetworkIO){new Probe }} - val release = (new ClientSourcedIO){(new LogicalNetworkIO){new Release }} - val release_data = (new ClientSourcedIO){(new LogicalNetworkIO){new ReleaseData }} - val grant = (new MasterSourcedIO){(new LogicalNetworkIO){new Grant }} - val grant_ack = (new ClientSourcedIO){(new LogicalNetworkIO){new GrantAck }} - override def clone = { new TileLinkIO().asInstanceOf[this.type] } -} } diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index 8ff26adc..51b64b16 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -79,8 +79,20 @@ class Probe extends PhysicalAddress { val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) } -class Release extends Bundle { +object Release +{ + def apply(r_type: Bits, addr: UFix, client_xact_id: UFix, master_xact_id: UFix) = { + val rel = new Release + rel.r_type := r_type + rel.addr := addr + rel.client_xact_id := client_xact_id + rel.master_xact_id := master_xact_id + rel + } +} +class Release extends PhysicalAddress { val r_type = Bits(width = RELEASE_TYPE_MAX_BITS) + val client_xact_id = Bits(width = CLIENT_XACT_ID_BITS) val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) } @@ -90,7 +102,6 @@ class Grant extends MemData { val g_type = Bits(width = GRANT_TYPE_MAX_BITS) val client_xact_id = Bits(width = CLIENT_XACT_ID_BITS) val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) - val require_ack = Bool() } class GrantAck extends Bundle { diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 21aaba8c..a439c7b6 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -288,7 +288,6 @@ class CoherenceHubNull(implicit conf: CoherenceHubConfiguration) extends Coheren grant.bits.payload.client_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, acquire.bits.payload.client_xact_id) grant.bits.payload.master_xact_id := UFix(0) // don't care grant.bits.payload.data := io.mem.resp.bits.data - grant.bits.payload.require_ack := Bool(true) grant.valid := io.mem.resp.valid || acquire.valid && is_write && io.mem.req_cmd.ready io.tiles(0).abort.valid := Bool(false) @@ -368,7 +367,6 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co rep.bits.payload.client_xact_id := UFix(0) rep.bits.payload.master_xact_id := UFix(0) rep.bits.payload.data := io.mem.resp.bits.data - rep.bits.payload.require_ack := Bool(true) rep.valid := Bool(false) when(io.mem.resp.valid && (UFix(j) === init_client_id_arr(mem_idx))) { rep.bits.payload.g_type := co.getGrantType(a_type_arr(mem_idx), sh_count_arr(mem_idx)) @@ -537,11 +535,11 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren { implicit val lnConf = conf.ln val co = conf.co - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) + val trackerList = new WritebackTracker(0) +: (1 to NGLOBAL_XACTS).map(new AcquireTracker(_)) val release_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth must >= NPRIMARY val acquire_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth should >= NPRIMARY - for( i <- 0 until NGLOBAL_XACTS ) { + for( i <- 0 to NGLOBAL_XACTS ) { val t = trackerList(i) t.io.tile_incoherent := io.incoherent.toBits t.io.mem_resp.valid := io.mem.resp.valid && (io.mem.resp.bits.tag === UFix(i)) @@ -559,12 +557,12 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } val abort_state = Reg(resetVal = s_idle) val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) - val any_conflict = trackerList.map(_.io.has_conflict).reduce(_||_) + val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val all_busy = trackerList.map(_.io.busy).reduce(_&&_) - val want_to_abort = acquire.valid && (any_conflict || all_busy || (!acquire_data_dep_q.io.enq.ready && co.messageHasData(acquire.bits.payload))) + val want_to_abort = acquire.valid && (any_acquire_conflict || all_busy || (!acquire_data_dep_q.io.enq.ready && co.messageHasData(acquire.bits.payload))) - val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } - for( i <- 0 until NGLOBAL_XACTS ) { + val alloc_arb = (new Arbiter(NGLOBAL_XACTS+1)) { Bool() } + for( i <- 0 to NGLOBAL_XACTS ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy trackerList(i).io.acquire.bits := acquire.bits trackerList(i).io.acquire.valid := (abort_state === s_idle) && !want_to_abort && acquire.valid && alloc_arb.io.in(i).ready @@ -610,8 +608,8 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren } // Handle probe request generation - val probe_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new Probe }} - for( i <- 0 until NGLOBAL_XACTS ) { + val probe_arb = (new Arbiter(NGLOBAL_XACTS+1)){(new LogicalNetworkIO){ new Probe }} + for( i <- 0 to NGLOBAL_XACTS ) { val t = trackerList(i).io probe_arb.io.in(i).bits := t.probe.bits probe_arb.io.in(i).valid := t.probe.valid @@ -622,13 +620,16 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren // Handle probe replies, which may or may not have data val release = io.network.release val release_data = io.network.release_data - val idx = release.bits.payload.master_xact_id + val voluntary = co.isVoluntary(release.bits.payload) + val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) + val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)){Bool()}.lastIndexWhere{b: Bool => b} + val idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UFix(0)), release.bits.payload.master_xact_id) release.ready := trackerList.map(_.io.release.ready).reduce(_||_) release_data.ready := trackerList.map(_.io.release_data.ready).reduce(_||_) release_data_dep_q.io.enq.valid := release.valid && co.messageHasData(release.bits.payload) - release_data_dep_q.io.enq.bits.master_xact_id := release.bits.payload.master_xact_id + release_data_dep_q.io.enq.bits.master_xact_id := idx release_data_dep_q.io.deq.ready := trackerList.map(_.io.release_data_dep.ready).reduce(_||_) - for( i <- 0 until NGLOBAL_XACTS ) { + for( i <- 0 to NGLOBAL_XACTS ) { trackerList(i).io.release_data.valid := release_data.valid trackerList(i).io.release_data.bits := release_data.bits trackerList(i).io.release_data_dep.valid := release_data_dep_q.io.deq.valid @@ -639,28 +640,27 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren // Reply to initial requestor // Forward memory responses from mem to tile or arbitrate to ack - val grant_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new Grant }} - for( i <- 0 until NGLOBAL_XACTS ) { + val grant_arb = (new Arbiter(NGLOBAL_XACTS+1)){(new LogicalNetworkIO){ new Grant }} + for( i <- 0 to NGLOBAL_XACTS ) { val t = trackerList(i).io grant_arb.io.in(i).bits := t.grant.bits grant_arb.io.in(i).valid := t.grant.valid t.grant.ready := grant_arb.io.in(i).ready } - grant_arb.io.out.ready := Bool(false) io.network.grant.valid := grant_arb.io.out.valid io.network.grant.bits := grant_arb.io.out.bits grant_arb.io.out.ready := io.network.grant.ready when(io.mem.resp.valid) { io.network.grant.valid := Bool(true) io.network.grant.bits := Vec(trackerList.map(_.io.grant.bits)){(new LogicalNetworkIO){new Grant}}(io.mem.resp.bits.tag) - for( i <- 0 until NGLOBAL_XACTS ) { + for( i <- 0 to NGLOBAL_XACTS ) { trackerList(i).io.grant.ready := (io.mem.resp.bits.tag === UFix(i)) && io.network.grant.ready } } // Free finished transactions val ack = io.network.grant_ack - for( i <- 0 until NGLOBAL_XACTS ) { + for( i <- 0 to NGLOBAL_XACTS ) { trackerList(i).io.free := ack.valid && (ack.bits.payload.master_xact_id === UFix(i)) } ack.ready := Bool(true) @@ -668,9 +668,9 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests // and once we have picked a request, get the right write data - val mem_req_cmd_arb = (new Arbiter(NGLOBAL_XACTS)) { new MemReqCmd() } - val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemData() } - for( i <- 0 until NGLOBAL_XACTS ) { + val mem_req_cmd_arb = (new Arbiter(NGLOBAL_XACTS+1)) { new MemReqCmd() } + val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS+1)) { new MemData() } + for( i <- 0 to NGLOBAL_XACTS ) { mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock @@ -679,7 +679,8 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren io.mem.req_data <> Queue(mem_req_data_arb.io.out) } -class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Component { + +abstract class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Component with MemoryRequestGenerator { val co = conf.co implicit val ln = conf.ln val io = new Bundle { @@ -699,15 +700,83 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val probe = (new FIFOIO) {(new LogicalNetworkIO) { new Probe }} val grant = (new FIFOIO) {(new LogicalNetworkIO) { new Grant }} val busy = Bool(OUTPUT) - val has_conflict = Bool(OUTPUT) + val has_acquire_conflict = Bool(OUTPUT) + val has_release_conflict = Bool(OUTPUT) } +} +class WritebackTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends XactTracker(id)(conf) { + val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(4){ UFix() } + val state = Reg(resetVal = s_idle) + val xact = Reg{ new Release } + val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val release_data_needs_write = Reg(resetVal = Bool(false)) + val mem_cmd_sent = Reg(resetVal = Bool(false)) + + io.acquire.ready := Bool(false) + io.acquire_data.ready := Bool(false) + io.acquire_data_dep.ready := Bool(false) + io.mem_resp.ready := Bool(false) + io.probe.valid := Bool(false) + io.busy := Bool(true) + io.has_acquire_conflict := Bool(false) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.release.bits.payload.addr) && (state != s_idle) + + io.mem_req_cmd.valid := Bool(false) + io.mem_req_cmd.bits.rw := Bool(false) + io.mem_req_cmd.bits.addr := xact.addr + io.mem_req_cmd.bits.tag := UFix(id) + io.mem_req_data.valid := Bool(false) + io.mem_req_data.bits.data := UFix(0) + io.mem_req_lock := Bool(false) + io.release.ready := Bool(false) + io.release_data.ready := Bool(false) + io.release_data_dep.ready := Bool(false) + io.grant.valid := Bool(false) + io.grant.bits.payload.g_type := co.getGrantType(xact, UFix(0)) + io.grant.bits.payload.client_xact_id := xact.client_xact_id + io.grant.bits.payload.master_xact_id := UFix(id) + io.grant.bits.header.dst := init_client_id_ + + switch (state) { + is(s_idle) { + when( io.release.valid ) { + xact := io.release.bits.payload + init_client_id_ := io.release.bits.header.src + release_data_needs_write := co.messageHasData(io.release.bits.payload) + mem_cnt := UFix(0) + mem_cmd_sent := Bool(false) + io.release.ready := Bool(true) + state := s_mem + } + } + is(s_mem) { + when (release_data_needs_write) { + doMemReqWrite(io.mem_req_cmd, + io.mem_req_data, + io.mem_req_lock, + io.release_data, + release_data_needs_write, + mem_cmd_sent, + io.release_data_dep.ready, + io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(id))) + } . otherwise { state := s_ack } + } + is(s_ack) { + io.grant.valid := Bool(true) + when(io.grant.ready) { state := s_idle } + } + } +} + +class AcquireTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends XactTracker(id)(conf) { val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) val xact = Reg{ new Acquire } val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) //TODO: Will need id reg for merged release xacts val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt_) val release_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) val x_needs_read = Reg(resetVal = Bool(false)) @@ -715,9 +784,6 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val release_data_needs_write = Reg(resetVal = Bool(false)) val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) - val mem_cnt_next = mem_cnt + UFix(1) - val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) val probe_initial_flags = Bits(width = conf.ln.nTiles) probe_initial_flags := Bits(0) if (conf.ln.nTiles > 1) { @@ -730,10 +796,10 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val myflag = Mux(probe_self, Bits(0), UFixToOH(io.acquire.bits.header.src(log2Up(conf.ln.nTiles)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } - val all_grants_require_acks = Bool(true) io.busy := state != s_idle - io.has_conflict := co.isCoherenceConflict(xact.addr, io.acquire.bits.payload.addr) && (state != s_idle) + io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, io.acquire.bits.payload.addr) && (state != s_idle) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.release.bits.payload.addr) && (state != s_idle) io.mem_req_cmd.valid := Bool(false) io.mem_req_cmd.bits.rw := Bool(false) io.mem_req_cmd.bits.addr := xact.addr @@ -747,10 +813,9 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com io.probe.bits.payload.addr := xact.addr io.probe.bits.header.dst := UFix(0) io.grant.bits.payload.data := io.mem_resp.bits.data - io.grant.bits.payload.g_type := co.getGrantType(xact.a_type, init_sharer_cnt_) + io.grant.bits.payload.g_type := grant_type io.grant.bits.payload.client_xact_id := xact.client_xact_id io.grant.bits.payload.master_xact_id := UFix(id) - io.grant.bits.payload.require_ack := all_grants_require_acks io.grant.bits.header.dst := init_client_id_ io.grant.valid := (io.mem_resp.valid && (UFix(id) === io.mem_resp.bits.tag)) io.acquire.ready := Bool(false) @@ -821,12 +886,12 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { state := Mux(co.needsAckReply(xact.a_type, UFix(0)), s_ack, - Mux(all_grants_require_acks, s_busy, s_idle)) + Mux(co.requiresAck(io.grant.bits.payload), s_busy, s_idle)) } } is(s_ack) { io.grant.valid := Bool(true) - when(io.grant.ready) { state := Mux(all_grants_require_acks, s_busy, s_idle) } + when(io.grant.ready) { state := Mux(co.requiresAck(io.grant.bits.payload), s_busy, s_idle) } } is(s_busy) { // Nothing left to do but wait for transaction to complete when (io.free) { @@ -834,6 +899,12 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com } } } +} + +trait MemoryRequestGenerator { + val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) + val mem_cnt_next = mem_cnt + UFix(1) + val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) def doMemReqWrite[T <: Data](req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: FIFOIO[LogicalNetworkIO[T]], trigger: Bool, cmd_sent: Bool, pop_dep: Bool, at_front_of_dep_queue: Bool) { req_cmd.bits.rw := Bool(true) From 944f56a7660166933e74cbc4942d84e392905601 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 28 Feb 2013 14:55:19 -0800 Subject: [PATCH 161/374] remove duplicate definitions --- uncore/src/package.scala | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 46d46661..21ea9433 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -106,20 +106,4 @@ class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkCon val payload = data override def clone = { new LogicalNetworkIO()(data).asInstanceOf[this.type] } } - -abstract class DirectionalFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) -class ClientSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) -class MasterSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) {flip()} - -class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val acquire = (new ClientSourcedIO){(new LogicalNetworkIO){new Acquire }} - val acquire_data = (new ClientSourcedIO){(new LogicalNetworkIO){new AcquireData }} - val abort = (new MasterSourcedIO){(new LogicalNetworkIO){new Abort }} - val probe = (new MasterSourcedIO){(new LogicalNetworkIO){new Probe }} - val release = (new ClientSourcedIO){(new LogicalNetworkIO){new Release }} - val release_data = (new ClientSourcedIO){(new LogicalNetworkIO){new ReleaseData }} - val grant = (new MasterSourcedIO){(new LogicalNetworkIO){new Grant }} - val grant_ack = (new ClientSourcedIO){(new LogicalNetworkIO){new GrantAck }} - override def clone = { new TileLinkIO().asInstanceOf[this.type] } -} } From 9f0ccbeac58c32ae2416e030d53dc656969fb7a1 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 28 Feb 2013 18:13:41 -0800 Subject: [PATCH 162/374] writebacks on release network pass asm tests and bmarks --- uncore/src/coherence.scala | 213 +++++++++++++++++-------------------- 1 file changed, 100 insertions(+), 113 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index e1cc453e..effba2e9 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -28,20 +28,21 @@ abstract class CoherencePolicy { def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix - def getAcquireTypeOnCacheControl(cmd: Bits): Bits - def getAcquireTypeOnWriteback(): Bits + def getReleaseTypeOnCacheControl(cmd: Bits): Bits + def getReleaseTypeOnVoluntaryWriteback(): Bits def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix): Release - def newRelease (incoming: Probe, state: UFix): Release + def newRelease (incoming: Probe, state: UFix, id: UFix): Release - def messageHasData (reply: Release): Bool + def messageHasData (rel: Release): Bool def messageHasData (acq: Acquire): Bool - def messageHasData (reply: Grant): Bool + def messageHasData (grant: Grant): Bool def messageUpdatesDataArray (reply: Grant): Bool def messageIsUncached(acq: Acquire): Bool def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool def isVoluntary(rel: Release): Bool + def isVoluntary(gnt: Grant): Bool def getGrantType(a_type: UFix, count: UFix): Bits def getGrantType(rel: Release, count: UFix): Bits def getProbeType(a_type: UFix, global_state: UFix): UFix @@ -50,6 +51,9 @@ abstract class CoherencePolicy { def needsAckReply(a_type: UFix, global_state: UFix): Bool def requiresAck(grant: Grant): Bool def requiresAck(release: Release): Bool + def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool + + def uFixListContains(list: List[UFix], elem: UFix): Bool = list.map(elem === _).reduceLeft(_||_) } trait UncachedTransactions { @@ -66,13 +70,8 @@ abstract class CoherencePolicyWithUncached extends CoherencePolicy with Uncached abstract class IncoherentPolicy extends CoherencePolicy { // UNIMPLEMENTED def newStateOnProbe(incoming: Probe, state: UFix): Bits = state - def newRelease (incoming: Probe, state: UFix): Release = { - val reply = new Release - reply.r_type := UFix(0) - reply.master_xact_id := UFix(0) - reply - } - def messageHasData (reply: Release) = Bool(false) + def newRelease (incoming: Probe, state: UFix, id: UFix): Release = Release( UFix(0), UFix(0), UFix(0), UFix(0)) + def messageHasData (rel: Release) = Bool(false) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = Bool(false) def getGrantType(a_type: UFix, count: UFix): Bits = Bits(0) def getGrantType(rel: Release, count: UFix): Bits = Bits(0) @@ -82,15 +81,19 @@ abstract class IncoherentPolicy extends CoherencePolicy { def needsAckReply(a_type: UFix, global_state: UFix): Bool = Bool(false) def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = Bool(false) + } class ThreeStateIncoherence extends IncoherentPolicy { val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(3){ UFix() } - val grantData :: grantAck :: Nil = Enum(2){ UFix() } + val grantVoluntaryAck :: grantData :: grantAck :: Nil = Enum(3){ UFix() } val releaseVoluntaryInvalidateData :: releaseInvalidateAck :: Nil = Enum(2){ UFix() } - val uncachedTypeList = List() - val hasDataTypeList = List(acquireWriteback) + val uncachedAcquireTypeList = List() + val hasDataAcquireTypeList = List(acquireWriteback) + val hasDataReleaseTypeList = List(acquireWriteback) + val hasDataGrantTypeList = List(grantData) def isHit ( cmd: Bits, state: UFix): Bool = (state === tileClean || state === tileDirty) def isValid (state: UFix): Bool = state != tileInvalid @@ -116,6 +119,7 @@ class ThreeStateIncoherence extends IncoherentPolicy { def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData + def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -125,13 +129,13 @@ class ThreeStateIncoherence extends IncoherentPolicy { val (read, write) = cpuCmdToRW(cmd) Mux(write, acquireReadDirty, outstanding.a_type) } - def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteback //TODO - def getAcquireTypeOnWriteback(): Bits = acquireWriteback + def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): Bits = releaseVoluntaryInvalidateData - def messageHasData (init: Acquire): Bool = hasDataTypeList.map(t => init.a_type === t).reduceLeft(_||_) - def messageHasData (reply: Grant) = (reply.g_type === grantData) + def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) + def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) def messageUpdatesDataArray (reply: Grant) = (reply.g_type === grantData) - def messageIsUncached(init: Acquire): Bool = uncachedTypeList.map(t => init.a_type === t).reduceLeft(_||_) + def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) } class MICoherence extends CoherencePolicyWithUncached { @@ -143,8 +147,10 @@ class MICoherence extends CoherencePolicyWithUncached { val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(7){ UFix() } val probeInvalidate :: probeCopy :: Nil = Enum(2){ UFix() } val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(5){ UFix() } - val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, grantReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseCopyData) + val hasDataGrantTypeList = List(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = state != tileInvalid def isValid (state: UFix): Bool = state != tileInvalid @@ -194,14 +200,14 @@ class MICoherence extends CoherencePolicyWithUncached { def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData + def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = acquireReadExclusive def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = acquireReadExclusive - def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteUncached - def getAcquireTypeOnWriteback(): Bits = getAcquireTypeOnCacheControl(M_INV) + def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def newRelease (incoming: Probe, state: UFix): Release = { - val reply = new Release + def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeCopy -> releaseCopyData @@ -210,23 +216,16 @@ class MICoherence extends CoherencePolicyWithUncached { probeInvalidate -> releaseInvalidateAck, probeCopy -> releaseCopyAck )) - reply.r_type := Mux(needsWriteback(state), with_data, without_data) - reply.master_xact_id := incoming.master_xact_id - reply + Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) } - def messageHasData (reply: Release): Bool = { - (reply.r_type === releaseInvalidateData || - reply.r_type === releaseCopyData) - } - def messageHasData (acq: Acquire): Bool = hasDataTypeList.map(t => acq.a_type === t).reduceLeft(_||_) - def messageHasData (reply: Grant): Bool = { - (reply.g_type != grantWriteUncached && reply.g_type != grantWriteWordUncached) - } + def messageHasData (rel: Release): Bool = uFixListContains(hasDataReleaseTypeList, rel.r_type) + def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) + def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uncachedTypeList.map(t => acq.a_type === t).reduceLeft(_||_) + def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) @@ -269,6 +268,7 @@ class MICoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } class MEICoherence extends CoherencePolicyWithUncached { @@ -280,8 +280,11 @@ class MEICoherence extends CoherencePolicyWithUncached { val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(8){ UFix() } val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(7){ UFix() } - val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, grantReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + + val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val hasDataGrantTypeList = List(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = state != tileInvalid def isValid (state: UFix): Bool = state != tileInvalid @@ -340,6 +343,7 @@ class MEICoherence extends CoherencePolicyWithUncached { def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData + def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -349,11 +353,10 @@ class MEICoherence extends CoherencePolicyWithUncached { val (read, write) = cpuCmdToRW(cmd) Mux(write, acquireReadExclusiveDirty, outstanding.a_type) } - def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteUncached - def getAcquireTypeOnWriteback(): Bits = getAcquireTypeOnCacheControl(M_INV) + def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def newRelease (incoming: Probe, state: UFix): Release = { - val reply = new Release + def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -364,24 +367,16 @@ class MEICoherence extends CoherencePolicyWithUncached { probeDowngrade -> releaseDowngradeAck, probeCopy -> releaseCopyAck )) - reply.r_type := Mux(needsWriteback(state), with_data, without_data) - reply.master_xact_id := incoming.master_xact_id - reply + Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) } - def messageHasData (reply: Release): Bool = { - (reply.r_type === releaseInvalidateData || - reply.r_type === releaseDowngradeData || - reply.r_type === releaseCopyData) - } - def messageHasData (acq: Acquire): Bool = hasDataTypeList.map(t => acq.a_type === t).reduceLeft(_||_) - def messageHasData (reply: Grant): Bool = { - (reply.g_type != grantWriteUncached && reply.g_type != grantReadExclusiveAck && reply.g_type != grantWriteWordUncached) - } + def messageHasData (rel: Release): Bool = uFixListContains(hasDataReleaseTypeList, rel.r_type) + def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) + def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadExclusive) } - def messageIsUncached(init: Acquire): Bool = uncachedTypeList.map(t => init.a_type === t).reduceLeft(_||_) + def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) @@ -426,6 +421,8 @@ class MEICoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + + def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } class MSICoherence extends CoherencePolicyWithUncached { @@ -437,8 +434,10 @@ class MSICoherence extends CoherencePolicyWithUncached { val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(9){ UFix() } val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(7){ UFix() } - val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, grantReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) @@ -504,6 +503,7 @@ class MSICoherence extends CoherencePolicyWithUncached { def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData + def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -513,11 +513,10 @@ class MSICoherence extends CoherencePolicyWithUncached { val (read, write) = cpuCmdToRW(cmd) Mux(write, acquireReadExclusive, outstanding.a_type) } - def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteUncached - def getAcquireTypeOnWriteback(): Bits = getAcquireTypeOnCacheControl(M_INV) + def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def newRelease (incoming: Probe, state: UFix): Release = { - val reply = new Release + def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -528,24 +527,16 @@ class MSICoherence extends CoherencePolicyWithUncached { probeDowngrade -> releaseDowngradeAck, probeCopy -> releaseCopyAck )) - reply.r_type := Mux(needsWriteback(state), with_data, without_data) - reply.master_xact_id := incoming.master_xact_id - reply + Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) } - def messageHasData (reply: Release): Bool = { - (reply.r_type === releaseInvalidateData || - reply.r_type === releaseDowngradeData || - reply.r_type === releaseCopyData) - } - def messageHasData (acq: Acquire): Bool = hasDataTypeList.map(t => acq.a_type === t).reduceLeft(_||_) - def messageHasData (reply: Grant): Bool = { - (reply.g_type != grantWriteUncached && reply.g_type != grantReadExclusiveAck && reply.g_type != grantWriteWordUncached) - } + def messageHasData (rel: Release): Bool = uFixListContains(hasDataReleaseTypeList, rel.r_type) + def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) + def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uncachedTypeList.map(t => acq.a_type === t).reduceLeft(_||_) + def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) @@ -587,6 +578,8 @@ class MSICoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + + def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } class MESICoherence extends CoherencePolicyWithUncached { @@ -598,8 +591,11 @@ class MESICoherence extends CoherencePolicyWithUncached { val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(9){ UFix() } val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(7){ UFix() } - val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + + val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) @@ -665,6 +661,7 @@ class MESICoherence extends CoherencePolicyWithUncached { def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData + def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -674,11 +671,10 @@ class MESICoherence extends CoherencePolicyWithUncached { val (read, write) = cpuCmdToRW(cmd) Mux(write, acquireReadExclusive, outstanding.a_type) } - def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteUncached - def getAcquireTypeOnWriteback(): Bits = getAcquireTypeOnCacheControl(M_INV) + def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def newRelease (incoming: Probe, state: UFix): Release = { - val reply = new Release + def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -689,24 +685,16 @@ class MESICoherence extends CoherencePolicyWithUncached { probeDowngrade -> releaseDowngradeAck, probeCopy -> releaseCopyAck )) - reply.r_type := Mux(needsWriteback(state), with_data, without_data) - reply.master_xact_id := incoming.master_xact_id - reply + Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) } - def messageHasData (reply: Release): Bool = { - (reply.r_type === releaseInvalidateData || - reply.r_type === releaseDowngradeData || - reply.r_type === releaseCopyData) - } - def messageHasData (acq: Acquire): Bool = hasDataTypeList.map(t => acq.a_type === t).reduceLeft(_||_) - def messageHasData (reply: Grant): Bool = { - (reply.g_type != grantWriteUncached && reply.g_type != grantReadExclusiveAck && reply.g_type != grantWriteWordUncached) - } + def messageHasData (rel: Release): Bool = uFixListContains(hasDataReleaseTypeList, rel.r_type) + def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) + def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uncachedTypeList.map(t => acq.a_type === t).reduceLeft(_||_) + def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) @@ -752,6 +740,8 @@ class MESICoherence extends CoherencePolicyWithUncached { def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + + def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } class MigratoryCoherence extends CoherencePolicyWithUncached { @@ -762,10 +752,11 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(10){ UFix() } val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(4){ UFix() } val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(11){ UFix() } - val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) - def uFixListContains(list: List[UFix], elem: UFix): Bool = list.map(elem === _).reduceLeft(_||_) + val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadMigratory, grantReadWordUncached, grantAtomicUncached) + val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) def isHit (cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) @@ -844,6 +835,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData + def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -853,12 +845,11 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { val (read, write) = cpuCmdToRW(cmd) Mux(write, Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), outstanding.a_type) } - def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteUncached - def getAcquireTypeOnWriteback(): Bits = getAcquireTypeOnCacheControl(M_INV) + def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def newRelease (incoming: Probe, state: UFix): Release = { + def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { Assert( incoming.p_type === probeInvalidateOthers && needsWriteback(state), "Bad probe request type, should be impossible.") - val reply = new Release() val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> Mux(uFixListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), releaseInvalidateDataMigratory, releaseInvalidateData), @@ -871,22 +862,16 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { probeDowngrade -> Mux(state != tileInvalid, releaseDowngradeAckHasCopy, releaseDowngradeAck), probeCopy -> releaseCopyAck )) - reply.r_type := Mux(needsWriteback(state), with_data, without_data) - reply.master_xact_id := incoming.master_xact_id - reply + Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) } - def messageHasData (reply: Release): Bool = { - uFixListContains(List(releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory), reply.r_type) - } - def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataTypeList, acq.a_type) - def messageHasData (reply: Grant): Bool = { - uFixListContains(List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadMigratory, grantReadWordUncached, grantAtomicUncached), reply.g_type) - } + def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) + def messageHasData (rel: Release): Bool = uFixListContains(hasDataReleaseTypeList, rel.r_type) + def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) def messageUpdatesDataArray (reply: Grant): Bool = { uFixListContains(List(grantReadShared, grantReadExclusive, grantReadMigratory), reply.g_type) } - def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) @@ -933,4 +918,6 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + + def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } From 717a78f96435b51de9e99a2b9d0f3f3964d93fda Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 19 Mar 2013 00:41:09 -0700 Subject: [PATCH 163/374] fix seqRead inference --- uncore/src/llc.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index 7ccdedb8..55d543b7 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -40,7 +40,7 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[Bit for (j <- 0 until nWide) { val mem = leaf.clone var dout: Bits = null - val dout1 = if (postLatency > 0) Reg() { Bits() } else null + val ridx = if (postLatency > 0) Reg() { Bits() } else null var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) if (colMux > 1) @@ -48,15 +48,15 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[Bit val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) when (in.valid) { when (in.bits.rw) { mem.write(idx, wdata0, wmask0) } - .otherwise { if (postLatency > 0) dout1 := mem(idx) } + .otherwise { if (postLatency > 0) ridx := idx } } if (postLatency == 0) { dout = mem(idx) } else if (postLatency == 1) { - dout = dout1 + dout = mem(ridx) } else - dout = Pipe(reg_ren, dout1, postLatency-1).bits + dout = Pipe(reg_ren, mem(ridx), postLatency-1).bits rdata(j) := dout } From f120800aa259592400793f7dfa1b15a141f82e6c Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 19 Mar 2013 00:41:28 -0700 Subject: [PATCH 164/374] add DRAMSideLLCNull --- uncore/src/llc.scala | 96 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index 55d543b7..1f9f4b2a 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -442,3 +442,99 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da io.mem.req_cmd <> memCmdArb.io.out io.mem.req_data <> writeback.io.mem.req_data } + +class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Component +{ + val io = new ioQueue(entries)(data) + require(isPow2(entries) && entries > 1) + + val do_flow = Bool() + val do_enq = io.enq.fire() && !do_flow + val do_deq = io.deq.fire() && !do_flow + + val maybe_full = Reg(resetVal = Bool(false)) + val enq_ptr = Counter(do_enq, entries)._1 + val deq_ptr = Counter(do_deq, entries)._1 + when (do_enq != do_deq) { maybe_full := do_enq } + + val ptr_match = enq_ptr === deq_ptr + val empty = ptr_match && !maybe_full + val full = ptr_match && maybe_full + val atLeastTwo = full || enq_ptr - deq_ptr >= UFix(2) + do_flow := empty && io.deq.ready + + val ram = Mem(entries, seqRead = true){Bits(width = data.getWidth)} + val ram_addr = Reg{Bits()} + val ram_out_valid = Reg{Bool()} + ram_out_valid := Bool(false) + when (do_enq) { ram(enq_ptr) := io.enq.bits.toBits } + when (io.deq.ready && (atLeastTwo || !io.deq.valid && !empty)) { + ram_out_valid := Bool(true) + ram_addr := Mux(io.deq.valid, deq_ptr + UFix(1), deq_ptr) + } + + io.deq.valid := Mux(empty, io.enq.valid, ram_out_valid) + io.enq.ready := !full + io.deq.bits := Mux(empty, io.enq.bits, data.fromBits(ram(ram_addr))) +} + +class HellaQueue[T <: Data](val entries: Int)(data: => T) extends Component +{ + val io = new ioQueue(entries)(data) + + val fq = new HellaFlowQueue(entries)(data) + io.enq <> fq.io.enq + io.deq <> Queue(fq.io.deq, 1, pipe = true) +} + +object HellaQueue +{ + def apply[T <: Data](enq: FIFOIO[T], entries: Int) = { + val q = (new HellaQueue(entries)) { enq.bits.clone } + q.io.enq.valid := enq.valid // not using <> so that override is allowed + q.io.enq.bits := enq.bits + enq.ready := q.io.enq.ready + q.io.deq + } +} + +class DRAMSideLLCNull(numRequests: Int, refillCycles: Int) extends Component +{ + val io = new Bundle { + val cpu = new ioMem().flip + val mem = new ioMemPipe + } + + val numEntries = numRequests * refillCycles + val size = log2Down(numEntries) + 1 + + val inc = Bool() + val dec = Bool() + val count = Reg(resetVal = UFix(numEntries, size)) + val watermark = count >= UFix(refillCycles) + + when (inc && !dec) { + count := count + UFix(1) + } + when (!inc && dec) { + count := count - UFix(refillCycles) + } + when (inc && dec) { + count := count - UFix(refillCycles-1) + } + + val cmdq_mask = io.cpu.req_cmd.bits.rw || watermark + + io.mem.req_cmd.valid := io.cpu.req_cmd.valid && cmdq_mask + io.cpu.req_cmd.ready := io.mem.req_cmd.ready && cmdq_mask + io.mem.req_cmd.bits := io.cpu.req_cmd.bits + + io.mem.req_data <> io.cpu.req_data + + val resp_dataq = (new HellaQueue(numEntries)) { new MemResp } + resp_dataq.io.enq <> io.mem.resp + io.cpu.resp <> resp_dataq.io.deq + + inc := resp_dataq.io.deq.fire() + dec := io.mem.req_cmd.fire() && !io.mem.req_cmd.bits.rw +} From 7b019cb0da4f2699e9b1fc45b8f21fc7adf9c96d Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 19 Mar 2013 15:30:23 -0700 Subject: [PATCH 165/374] rmeove aborts --- uncore/src/tilelink.scala | 5 -- uncore/src/uncore.scala | 96 ++++----------------------------------- 2 files changed, 8 insertions(+), 93 deletions(-) diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index 51b64b16..73bf8aff 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -70,10 +70,6 @@ object Acquire class AcquireData extends MemData -class Abort extends Bundle { - val client_xact_id = Bits(width = CLIENT_XACT_ID_BITS) -} - class Probe extends PhysicalAddress { val p_type = Bits(width = PROBE_TYPE_MAX_BITS) val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) @@ -115,7 +111,6 @@ class MasterSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { val acquire = (new ClientSourcedIO){(new LogicalNetworkIO){new Acquire }} val acquire_data = (new ClientSourcedIO){(new LogicalNetworkIO){new AcquireData }} - val abort = (new MasterSourcedIO){(new LogicalNetworkIO){new Abort }} val probe = (new MasterSourcedIO){(new LogicalNetworkIO){new Probe }} val release = (new ClientSourcedIO){(new LogicalNetworkIO){new Release }} val release_data = (new ClientSourcedIO){(new LogicalNetworkIO){new ReleaseData }} diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index a439c7b6..8d1d396d 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -290,7 +290,6 @@ class CoherenceHubNull(implicit conf: CoherenceHubConfiguration) extends Coheren grant.bits.payload.data := io.mem.resp.bits.data grant.valid := io.mem.resp.valid || acquire.valid && is_write && io.mem.req_cmd.ready - io.tiles(0).abort.valid := Bool(false) io.tiles(0).grant_ack.ready := Bool(true) io.tiles(0).probe.valid := Bool(false) io.tiles(0).release.ready := Bool(true) @@ -427,51 +426,6 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co release_cnt_dec_arr(i)(j) := release.valid && (release.bits.payload.master_xact_id === UFix(i)) } } - - // Nack conflicting transaction init attempts - val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } - val abort_state_arr = Vec(conf.ln.nTiles) { Reg(resetVal = s_idle) } - val want_to_abort_arr = Vec(conf.ln.nTiles) { Bool() } - for( j <- 0 until conf.ln.nTiles ) { - val acquire = io.tiles(j).acquire - val acquire_data = io.tiles(j).acquire_data - val x_abort = io.tiles(j).abort - val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) - val conflicts = Vec(NGLOBAL_XACTS) { Bool() } - for( i <- 0 until NGLOBAL_XACTS) { - val t = trackerList(i).io - conflicts(i) := t.busy && acquire.valid && co.isCoherenceConflict(t.addr, acquire.bits.payload.addr) - } - x_abort.bits.payload.client_xact_id := acquire.bits.payload.client_xact_id - want_to_abort_arr(j) := acquire.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!acquire_data_dep_list(j).io.enq.ready && co.messageHasData(acquire.bits.payload))) - - x_abort.valid := Bool(false) - switch(abort_state_arr(j)) { - is(s_idle) { - when(want_to_abort_arr(j)) { - when(co.messageHasData(acquire.bits.payload)) { - abort_state_arr(j) := s_abort_drain - } . otherwise { - abort_state_arr(j) := s_abort_send - } - } - } - is(s_abort_drain) { // raises acquire_data.ready below - when(acquire_data.valid) { - abort_cnt := abort_cnt + UFix(1) - when(abort_cnt === ~UFix(0, width = log2Up(REFILL_CYCLES))) { - abort_state_arr(j) := s_abort_send - } - } - } - is(s_abort_send) { // nothing is dequeued for now - x_abort.valid := Bool(true) - when(x_abort.ready) { // raises acquire.ready below - abort_state_arr(j) := s_idle - } - } - } - } // Handle transaction initiation requests // Only one allocation per cycle @@ -493,16 +447,14 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val acquire = io.tiles(j).acquire val acquire_data = io.tiles(j).acquire_data val acquire_data_dep = acquire_data_dep_list(j).io.deq - val x_abort = io.tiles(j).abort - init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && acquire.valid + init_arb.io.in(j).valid := acquire.valid init_arb.io.in(j).bits.acquire := acquire.bits.payload init_arb.io.in(j).bits.client_id := UFix(j) val pop_acquires = trackerList.map(_.io.pop_acquire(j).toBool) - val do_pop = foldR(pop_acquires)(_||_) - acquire_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(acquire.bits.payload) && (abort_state_arr(j) === s_idle) + acquire.ready := foldR(pop_acquires)(_||_) + acquire_data_dep_list(j).io.enq.valid := acquire.ready && co.messageHasData(acquire.bits.payload) acquire_data_dep_list(j).io.enq.bits.master_xact_id := OHToUFix(pop_acquires) - acquire.ready := (x_abort.valid && x_abort.ready) || do_pop - acquire_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_acquire_data(j).toBool))(_||_) + acquire_data.ready := foldR(trackerList.map(_.io.pop_acquire_data(j).toBool))(_||_) acquire_data_dep.ready := foldR(trackerList.map(_.io.pop_acquire_dep(j).toBool))(_||_) } @@ -552,61 +504,29 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren // Init requests may or may not have data val acquire = io.network.acquire val acquire_data = io.network.acquire_data - val x_abort = io.network.abort val x_dep_deq = acquire_data_dep_q.io.deq - val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } - val abort_state = Reg(resetVal = s_idle) - val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val all_busy = trackerList.map(_.io.busy).reduce(_&&_) - val want_to_abort = acquire.valid && (any_acquire_conflict || all_busy || (!acquire_data_dep_q.io.enq.ready && co.messageHasData(acquire.bits.payload))) val alloc_arb = (new Arbiter(NGLOBAL_XACTS+1)) { Bool() } for( i <- 0 to NGLOBAL_XACTS ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy trackerList(i).io.acquire.bits := acquire.bits - trackerList(i).io.acquire.valid := (abort_state === s_idle) && !want_to_abort && acquire.valid && alloc_arb.io.in(i).ready + trackerList(i).io.acquire.valid := acquire.valid && alloc_arb.io.in(i).ready trackerList(i).io.acquire_data.bits := acquire_data.bits trackerList(i).io.acquire_data.valid := acquire_data.valid trackerList(i).io.acquire_data_dep.bits := x_dep_deq.bits trackerList(i).io.acquire_data_dep.valid := x_dep_deq.valid } - val pop_acquire = trackerList.map(_.io.acquire.ready).reduce(_||_) - acquire.ready := (x_abort.valid && x_abort.ready) || pop_acquire - acquire_data.ready := (abort_state === s_abort_drain) || trackerList.map(_.io.acquire_data.ready).reduce(_||_) - acquire_data_dep_q.io.enq.valid := pop_acquire && co.messageHasData(acquire.bits.payload) && (abort_state === s_idle) + acquire.ready := trackerList.map(_.io.acquire.ready).reduce(_||_) + acquire_data.ready := trackerList.map(_.io.acquire_data.ready).reduce(_||_) + acquire_data_dep_q.io.enq.valid := acquire.ready && co.messageHasData(acquire.bits.payload) acquire_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(trackerList.map(_.io.acquire.ready)) x_dep_deq.ready := trackerList.map(_.io.acquire_data_dep.ready).reduce(_||_) alloc_arb.io.out.ready := acquire.valid - // Nack conflicting transaction init attempts - x_abort.bits.header.dst := acquire.bits.header.src - x_abort.bits.payload.client_xact_id := acquire.bits.payload.client_xact_id - x_abort.valid := Bool(false) - switch(abort_state) { - is(s_idle) { - when(want_to_abort) { - abort_state := Mux( co.messageHasData(acquire.bits.payload), s_abort_drain, s_abort_send) - } - } - is(s_abort_drain) { // raises acquire_data.ready below - when(acquire_data.valid) { - abort_cnt := abort_cnt + UFix(1) - when(abort_cnt === ~UFix(0, width = log2Up(REFILL_CYCLES))) { - abort_state := s_abort_send - } - } - } - is(s_abort_send) { // nothing is dequeued for now - x_abort.valid := Bool(true) - when(x_abort.ready) { // raises acquire.ready - abort_state := s_idle - } - } - } - // Handle probe request generation val probe_arb = (new Arbiter(NGLOBAL_XACTS+1)){(new LogicalNetworkIO){ new Probe }} for( i <- 0 to NGLOBAL_XACTS ) { From a7ae7e5758eea704ed284667df0295e9b304a30c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 28 Feb 2013 19:49:05 -0800 Subject: [PATCH 166/374] Cleaned up self-probes --- uncore/src/coherence.scala | 7 +++++++ uncore/src/uncore.scala | 11 +++-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index effba2e9..f7a75afc 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -49,6 +49,7 @@ abstract class CoherencePolicy { def needsMemRead(a_type: UFix, global_state: UFix): Bool def needsMemWrite(a_type: UFix, global_state: UFix): Bool def needsAckReply(a_type: UFix, global_state: UFix): Bool + def needsSelfProbe(acq: Acquire): Bool def requiresAck(grant: Grant): Bool def requiresAck(release: Release): Bool def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool @@ -79,6 +80,7 @@ abstract class IncoherentPolicy extends CoherencePolicy { def needsMemRead(a_type: UFix, global_state: UFix): Bool = Bool(false) def needsMemWrite(a_type: UFix, global_state: UFix): Bool = Bool(false) def needsAckReply(a_type: UFix, global_state: UFix): Bool = Bool(false) + def needsSelfProbe(acq: Acquire) = Bool(false) def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = Bool(false) @@ -268,6 +270,7 @@ class MICoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + def needsSelfProbe(acq: Acquire) = acq.a_type === acquireReadUncached def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -421,6 +424,7 @@ class MEICoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + def needsSelfProbe(acq: Acquire) = acq.a_type === acquireReadUncached def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -578,6 +582,7 @@ class MSICoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + def needsSelfProbe(acq: Acquire) = acq.a_type === acquireReadUncached def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -740,6 +745,7 @@ class MESICoherence extends CoherencePolicyWithUncached { def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + def needsSelfProbe(acq: Acquire) = acq.a_type === acquireReadUncached def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -918,6 +924,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + def needsSelfProbe(acq: Acquire) = acq.a_type === acquireReadUncached def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 8d1d396d..e3ebbc07 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -105,14 +105,9 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val probe_initial_flags = Bits(width = conf.ln.nTiles) probe_initial_flags := Bits(0) if (conf.ln.nTiles > 1) { - // issue self-probes for uncached read xacts to facilitate I$ coherence - // TODO: this is hackish; figure out how to do it more systematically - val probe_self = co match { - case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.alloc_req.bits.acquire) - case _ => Bool(false) - } - val myflag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.client_id(log2Up(conf.ln.nTiles)-1,0))) - probe_initial_flags := ~(io.tile_incoherent | myflag) + val probe_self = co.needsSelfProbe(io.alloc_req.bits.acquire) + val probe_self_flag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.client_id(log2Up(conf.ln.nTiles)-1,0))) + probe_initial_flags := ~(io.tile_incoherent | probe_self_flag) } io.busy := state != s_idle From 319b4544d757f871a9fcd6437abf62847eeab179 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 28 Feb 2013 21:03:56 -0800 Subject: [PATCH 167/374] nTiles -> nClients in LogicalNetworkConfig --- uncore/src/package.scala | 2 +- uncore/src/uncore.scala | 116 +++++++++++++++++++-------------------- 2 files changed, 59 insertions(+), 59 deletions(-) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 8e511663..7cce3ff3 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -48,7 +48,7 @@ class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfi } } -case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nHubs: Int, nTiles: Int) +case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nMasters: Int, nClients: Int) abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[CoherenceAgentRole])(implicit conf: LogicalNetworkConfiguration) extends Component { val io: Vec[TileLinkType] diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index e3ebbc07..5609b701 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -24,9 +24,9 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val p_data = (new PipeIO) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) val grant_ack = Bool(INPUT) - val release_cnt_dec = Bits(INPUT, conf.ln.nTiles) - val probe_cnt_inc = Bits(INPUT, conf.ln.nTiles) - val tile_incoherent = Bits(INPUT, conf.ln.nTiles) + val release_cnt_dec = Bits(INPUT, conf.ln.nClients) + val probe_cnt_inc = Bits(INPUT, conf.ln.nClients) + val tile_incoherent = Bits(INPUT, conf.ln.nClients) val release_data = (new PipeIO) { new ReleaseData }.flip val acquire_data = (new PipeIO) { new AcquireData }.flip val sent_grant_ack = Bool(INPUT) @@ -44,13 +44,13 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val client_xact_id = Bits(OUTPUT, CLIENT_XACT_ID_BITS) val sharer_count = Bits(OUTPUT, conf.ln.idBits+1) val a_type = Bits(OUTPUT, ACQUIRE_TYPE_MAX_BITS) - val push_probe = Bits(OUTPUT, conf.ln.nTiles) - val pop_release = Bits(OUTPUT, conf.ln.nTiles) - val pop_release_data = Bits(OUTPUT, conf.ln.nTiles) - val pop_release_dep = Bits(OUTPUT, conf.ln.nTiles) - val pop_acquire = Bits(OUTPUT, conf.ln.nTiles) - val pop_acquire_data = Bits(OUTPUT, conf.ln.nTiles) - val pop_acquire_dep = Bits(OUTPUT, conf.ln.nTiles) + val push_probe = Bits(OUTPUT, conf.ln.nClients) + val pop_release = Bits(OUTPUT, conf.ln.nClients) + val pop_release_data = Bits(OUTPUT, conf.ln.nClients) + val pop_release_dep = Bits(OUTPUT, conf.ln.nClients) + val pop_acquire = Bits(OUTPUT, conf.ln.nClients) + val pop_acquire_data = Bits(OUTPUT, conf.ln.nClients) + val pop_acquire_dep = Bits(OUTPUT, conf.ln.nClients) val send_grant_ack = Bool(OUTPUT) } @@ -91,8 +91,8 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val state = Reg(resetVal = s_idle) val xact = Reg{ new Acquire } val init_client_id_ = Reg{ Bits() } - val release_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) - val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) + val release_count = if (conf.ln.nClients == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) + val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nClients)) val release_client_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) val acquire_data_needs_write = Reg(resetVal = Bool(false)) @@ -102,11 +102,11 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) - val probe_initial_flags = Bits(width = conf.ln.nTiles) + val probe_initial_flags = Bits(width = conf.ln.nClients) probe_initial_flags := Bits(0) - if (conf.ln.nTiles > 1) { + if (conf.ln.nClients > 1) { val probe_self = co.needsSelfProbe(io.alloc_req.bits.acquire) - val probe_self_flag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.client_id(log2Up(conf.ln.nTiles)-1,0))) + val probe_self_flag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.client_id(log2Up(conf.ln.nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | probe_self_flag) } @@ -115,7 +115,7 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex io.init_client_id := init_client_id_ io.release_client_id := release_client_id_ io.client_xact_id := xact.client_xact_id - io.sharer_count := UFix(conf.ln.nTiles) // TODO: Broadcast only + io.sharer_count := UFix(conf.ln.nClients) // TODO: Broadcast only io.a_type := xact.a_type io.mem_req_cmd.valid := Bool(false) @@ -129,13 +129,13 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex io.probe.bits.p_type := co.getProbeType(xact.a_type, UFix(0)) io.probe.bits.master_xact_id := UFix(id) io.probe.bits.addr := xact.addr - io.push_probe := Bits(0, width = conf.ln.nTiles) - io.pop_release := Bits(0, width = conf.ln.nTiles) - io.pop_release_data := Bits(0, width = conf.ln.nTiles) - io.pop_release_dep := Bits(0, width = conf.ln.nTiles) - io.pop_acquire := Bits(0, width = conf.ln.nTiles) - io.pop_acquire_data := Bits(0, width = conf.ln.nTiles) - io.pop_acquire_dep := Bits(0, width = conf.ln.nTiles) + io.push_probe := Bits(0, width = conf.ln.nClients) + io.pop_release := Bits(0, width = conf.ln.nClients) + io.pop_release_data := Bits(0, width = conf.ln.nClients) + io.pop_release_dep := Bits(0, width = conf.ln.nClients) + io.pop_acquire := Bits(0, width = conf.ln.nClients) + io.pop_acquire_data := Bits(0, width = conf.ln.nClients) + io.pop_acquire_dep := Bits(0, width = conf.ln.nClients) io.send_grant_ack := Bool(false) switch (state) { @@ -150,7 +150,7 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) io.pop_acquire := UFix(1) << io.alloc_req.bits.client_id - if(conf.ln.nTiles > 1) { + if(conf.ln.nClients > 1) { release_count := PopCount(probe_initial_flags) state := Mux(probe_initial_flags.orR, s_probe, s_mem) } else state := s_mem @@ -167,7 +167,7 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex when(io.release_cnt_dec.orR) { val dec = PopCount(io.release_cnt_dec) io.pop_release := io.release_cnt_dec - if(conf.ln.nTiles > 1) release_count := release_count - dec + if(conf.ln.nClients > 1) release_count := release_count - dec when(release_count === dec) { state := s_mem } @@ -223,7 +223,7 @@ case class CoherenceHubConfiguration(co: CoherencePolicy, ln: LogicalNetworkConf class CoherenceHubAdapter(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { val io = new Bundle { val net = (new TileLinkIO).flip - val hub = Vec(conf.nTiles) { new TileLinkIO } + val hub = Vec(conf.nClients) { new TileLinkIO } } val netClientProducedSubBundles = io.net.getClass.getMethods.filter( x => @@ -259,8 +259,8 @@ class CoherenceHubAdapter(implicit conf: LogicalNetworkConfiguration) extends Co abstract class CoherenceHub(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { val io = new Bundle { - val tiles = Vec(conf.nTiles) { new TileLinkIO }.flip - val incoherent = Vec(conf.nTiles) { Bool() }.asInput + val tiles = Vec(conf.nClients) { new TileLinkIO }.flip + val incoherent = Vec(conf.nClients) { Bool() }.asInput val mem = new ioMem } } @@ -307,8 +307,8 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val send_grant_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } val do_free_arr = Vec(NGLOBAL_XACTS){ Bool() } - val release_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } - val probe_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } + val release_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nClients){ Bool()} } + val probe_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nClients){ Bool()} } val sent_grant_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } val p_data_client_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } @@ -333,17 +333,17 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co sent_grant_ack_arr(i) := Bool(false) p_data_client_id_arr(i) := Bits(0, width = conf.ln.idBits) p_data_valid_arr(i) := Bool(false) - for( j <- 0 until conf.ln.nTiles) { + for( j <- 0 until conf.ln.nClients) { release_cnt_dec_arr(i)(j) := Bool(false) probe_cnt_inc_arr(i)(j) := Bool(false) } } - val release_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY - val acquire_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY + val release_data_dep_list = List.fill(conf.ln.nClients)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY + val acquire_data_dep_list = List.fill(conf.ln.nClients)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY // Free finished transactions - for( j <- 0 until conf.ln.nTiles ) { + for( j <- 0 until conf.ln.nClients ) { val ack = io.tiles(j).grant_ack when (ack.valid) { do_free_arr(ack.bits.payload.master_xact_id) := Bool(true) @@ -355,7 +355,7 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co // Forward memory responses from mem to tile or arbitrate to ack val mem_idx = io.mem.resp.bits.tag val ack_idx = PriorityEncoder(send_grant_ack_arr.toBits) - for( j <- 0 until conf.ln.nTiles ) { + for( j <- 0 until conf.ln.nClients ) { val rep = io.tiles(j).grant rep.bits.payload.g_type := UFix(0) rep.bits.payload.client_xact_id := UFix(0) @@ -393,7 +393,7 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co io.mem.req_data <> Queue(mem_req_data_arb.io.out) // Handle probe replies, which may or may not have data - for( j <- 0 until conf.ln.nTiles ) { + for( j <- 0 until conf.ln.nClients ) { val release = io.tiles(j).release val release_data = io.tiles(j).release_data val idx = release.bits.payload.master_xact_id @@ -413,10 +413,10 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co trackerList(i).io.release_data.valid := io.tiles(trackerList(i).io.release_client_id).release_data.valid trackerList(i).io.release_data.bits := io.tiles(trackerList(i).io.release_client_id).release_data.bits.payload - trackerList(i).io.release_data_dep.valid := MuxLookup(trackerList(i).io.release_client_id, release_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> release_data_dep_list(j).io.deq.valid)) - trackerList(i).io.release_data_dep.bits := MuxLookup(trackerList(i).io.release_client_id, release_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> release_data_dep_list(j).io.deq.bits)) + trackerList(i).io.release_data_dep.valid := MuxLookup(trackerList(i).io.release_client_id, release_data_dep_list(0).io.deq.valid, (0 until conf.ln.nClients).map( j => UFix(j) -> release_data_dep_list(j).io.deq.valid)) + trackerList(i).io.release_data_dep.bits := MuxLookup(trackerList(i).io.release_client_id, release_data_dep_list(0).io.deq.bits, (0 until conf.ln.nClients).map( j => UFix(j) -> release_data_dep_list(j).io.deq.bits)) - for( j <- 0 until conf.ln.nTiles) { + for( j <- 0 until conf.ln.nClients) { val release = io.tiles(j).release release_cnt_dec_arr(i)(j) := release.valid && (release.bits.payload.master_xact_id === UFix(i)) } @@ -426,7 +426,7 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co // Only one allocation per cycle // Init requests may or may not have data val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } - val init_arb = (new Arbiter(conf.ln.nTiles)) { new TrackerAllocReq } + val init_arb = (new Arbiter(conf.ln.nClients)) { new TrackerAllocReq } for( i <- 0 until NGLOBAL_XACTS ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready @@ -435,10 +435,10 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co trackerList(i).io.acquire_data.bits := io.tiles(trackerList(i).io.init_client_id).acquire_data.bits.payload trackerList(i).io.acquire_data.valid := io.tiles(trackerList(i).io.init_client_id).acquire_data.valid - trackerList(i).io.acquire_data_dep.bits := MuxLookup(trackerList(i).io.init_client_id, acquire_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> acquire_data_dep_list(j).io.deq.bits)) - trackerList(i).io.acquire_data_dep.valid := MuxLookup(trackerList(i).io.init_client_id, acquire_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> acquire_data_dep_list(j).io.deq.valid)) + trackerList(i).io.acquire_data_dep.bits := MuxLookup(trackerList(i).io.init_client_id, acquire_data_dep_list(0).io.deq.bits, (0 until conf.ln.nClients).map( j => UFix(j) -> acquire_data_dep_list(j).io.deq.bits)) + trackerList(i).io.acquire_data_dep.valid := MuxLookup(trackerList(i).io.init_client_id, acquire_data_dep_list(0).io.deq.valid, (0 until conf.ln.nClients).map( j => UFix(j) -> acquire_data_dep_list(j).io.deq.valid)) } - for( j <- 0 until conf.ln.nTiles ) { + for( j <- 0 until conf.ln.nClients ) { val acquire = io.tiles(j).acquire val acquire_data = io.tiles(j).acquire_data val acquire_data_dep = acquire_data_dep_list(j).io.deq @@ -457,8 +457,8 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co // Handle probe request generation // Must arbitrate for each request port - val probe_arb_arr = List.fill(conf.ln.nTiles)((new Arbiter(NGLOBAL_XACTS)) { new Probe() }) - for( j <- 0 until conf.ln.nTiles ) { + val probe_arb_arr = List.fill(conf.ln.nClients)((new Arbiter(NGLOBAL_XACTS)) { new Probe() }) + for( j <- 0 until conf.ln.nClients ) { for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io probe_arb_arr(j).io.in(i).bits := t.probe.bits @@ -473,7 +473,7 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co abstract class CoherenceAgent(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { val io = new Bundle { val network = (new TileLinkIO).flip - val incoherent = Vec(conf.nTiles) { Bool() }.asInput + val incoherent = Vec(conf.nClients) { Bool() }.asInput val mem = new ioMem } } @@ -604,7 +604,7 @@ abstract class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) ex val release = (new FIFOIO){(new LogicalNetworkIO) { new Release }}.flip val release_data = (new FIFOIO){(new LogicalNetworkIO) { new ReleaseData }}.flip val free = Bool(INPUT) - val tile_incoherent = Bits(INPUT, conf.ln.nTiles) + val tile_incoherent = Bits(INPUT, conf.ln.nClients) val release_data_dep = (new FIFOIO) { new TrackerDependency }.flip val acquire_data_dep = (new FIFOIO) { new TrackerDependency }.flip val mem_resp = (new FIFOIO) { new MemResp }.flip @@ -624,7 +624,7 @@ class WritebackTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extend val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(4){ UFix() } val state = Reg(resetVal = s_idle) val xact = Reg{ new Release } - val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) val release_data_needs_write = Reg(resetVal = Bool(false)) val mem_cmd_sent = Reg(resetVal = Bool(false)) @@ -688,27 +688,27 @@ class AcquireTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) val xact = Reg{ new Acquire } - val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) //TODO: Will need id reg for merged release xacts - val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt_) - val release_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) - val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) + val release_count = if (conf.ln.nClients == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) + val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nClients)) val x_needs_read = Reg(resetVal = Bool(false)) val acquire_data_needs_write = Reg(resetVal = Bool(false)) val release_data_needs_write = Reg(resetVal = Bool(false)) val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val probe_initial_flags = Bits(width = conf.ln.nTiles) + val probe_initial_flags = Bits(width = conf.ln.nClients) probe_initial_flags := Bits(0) - if (conf.ln.nTiles > 1) { + if (conf.ln.nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence // TODO: this is hackish; figure out how to do it more systematically val probe_self = co match { case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.acquire.bits.payload) case _ => Bool(false) } - val myflag = Mux(probe_self, Bits(0), UFixToOH(io.acquire.bits.header.src(log2Up(conf.ln.nTiles)-1,0))) + val myflag = Mux(probe_self, Bits(0), UFixToOH(io.acquire.bits.header.src(log2Up(conf.ln.nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } @@ -746,7 +746,7 @@ class AcquireTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends when( io.acquire.valid ) { xact := io.acquire.bits.payload init_client_id_ := io.acquire.bits.header.src - init_sharer_cnt_ := UFix(conf.ln.nTiles) // TODO: Broadcast only + init_sharer_cnt_ := UFix(conf.ln.nClients) // TODO: Broadcast only acquire_data_needs_write := co.messageHasData(io.acquire.bits.payload) x_needs_read := co.needsMemRead(io.acquire.bits.payload.a_type, UFix(0)) probe_flags := probe_initial_flags @@ -754,7 +754,7 @@ class AcquireTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) io.acquire.ready := Bool(true) - if(conf.ln.nTiles > 1) { + if(conf.ln.nClients > 1) { release_count := PopCount(probe_initial_flags) state := Mux(probe_initial_flags.orR, s_probe, s_mem) } else state := s_mem @@ -771,7 +771,7 @@ class AcquireTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends } when(io.release.valid) { io.release.ready := Bool(true) - if(conf.ln.nTiles > 1) release_count := release_count - UFix(1) + if(conf.ln.nClients > 1) release_count := release_count - UFix(1) when(release_count === UFix(1)) { state := s_mem } From c36b1dfa30290b5009925c719c99e243a7f6313a Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 20 Mar 2013 14:10:16 -0700 Subject: [PATCH 168/374] Cleaned up uncore and coherence interface. Removed defunct broadcast hub. Trait-ified tilelink bundle components. Added generalized mem arbiter. --- uncore/src/coherence.scala | 86 ++-- uncore/src/consts.scala | 6 +- uncore/src/package.scala | 10 +- uncore/src/tilelink.scala | 132 +++++- uncore/src/uncore.scala | 841 ++++++++----------------------------- 5 files changed, 345 insertions(+), 730 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index f7a75afc..b4c4067f 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -34,9 +34,7 @@ abstract class CoherencePolicy { def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix): Release def newRelease (incoming: Probe, state: UFix, id: UFix): Release - def messageHasData (rel: Release): Bool - def messageHasData (acq: Acquire): Bool - def messageHasData (grant: Grant): Bool + def messageHasData (rel: SourcedMessage): Bool def messageUpdatesDataArray (reply: Grant): Bool def messageIsUncached(acq: Acquire): Bool @@ -46,8 +44,8 @@ abstract class CoherencePolicy { def getGrantType(a_type: UFix, count: UFix): Bits def getGrantType(rel: Release, count: UFix): Bits def getProbeType(a_type: UFix, global_state: UFix): UFix - def needsMemRead(a_type: UFix, global_state: UFix): Bool - def needsMemWrite(a_type: UFix, global_state: UFix): Bool + def needsOuterRead(a_type: UFix, global_state: UFix): Bool + def needsOuterWrite(a_type: UFix, global_state: UFix): Bool def needsAckReply(a_type: UFix, global_state: UFix): Bool def needsSelfProbe(acq: Acquire): Bool def requiresAck(grant: Grant): Bool @@ -72,13 +70,12 @@ abstract class IncoherentPolicy extends CoherencePolicy { // UNIMPLEMENTED def newStateOnProbe(incoming: Probe, state: UFix): Bits = state def newRelease (incoming: Probe, state: UFix, id: UFix): Release = Release( UFix(0), UFix(0), UFix(0), UFix(0)) - def messageHasData (rel: Release) = Bool(false) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = Bool(false) def getGrantType(a_type: UFix, count: UFix): Bits = Bits(0) def getGrantType(rel: Release, count: UFix): Bits = Bits(0) def getProbeType(a_type: UFix, global_state: UFix): UFix = UFix(0) - def needsMemRead(a_type: UFix, global_state: UFix): Bool = Bool(false) - def needsMemWrite(a_type: UFix, global_state: UFix): Bool = Bool(false) + def needsOuterRead(a_type: UFix, global_state: UFix): Bool = Bool(false) + def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = Bool(false) def needsAckReply(a_type: UFix, global_state: UFix): Bool = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) def requiresAck(grant: Grant) = Bool(true) @@ -134,8 +131,12 @@ class ThreeStateIncoherence extends IncoherentPolicy { def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = releaseVoluntaryInvalidateData - def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) - def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) + def messageHasData( msg: SourcedMessage ) = msg match { + case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => Bool(false) + case _ => Bool(false) + } def messageUpdatesDataArray (reply: Grant) = (reply.g_type === grantData) def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) } @@ -221,9 +222,12 @@ class MICoherence extends CoherencePolicyWithUncached { Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) } - def messageHasData (rel: Release): Bool = uFixListContains(hasDataReleaseTypeList, rel.r_type) - def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) - def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) + def messageHasData(msg: SourcedMessage) = msg match { + case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uFixListContains(hasDataReleaseTypeList, rel.r_type) + case _ => Bool(false) + } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadExclusive) } @@ -259,10 +263,10 @@ class MICoherence extends CoherencePolicyWithUncached { )) } - def needsMemRead(a_type: UFix, global_state: UFix): Bool = { + def needsOuterRead(a_type: UFix, global_state: UFix): Bool = { (a_type != acquireWriteUncached) } - def needsMemWrite(a_type: UFix, global_state: UFix): Bool = { + def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } def needsAckReply(a_type: UFix, global_state: UFix): Bool = { @@ -373,9 +377,12 @@ class MEICoherence extends CoherencePolicyWithUncached { Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) } - def messageHasData (rel: Release): Bool = uFixListContains(hasDataReleaseTypeList, rel.r_type) - def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) - def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) + def messageHasData(msg: SourcedMessage) = msg match { + case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uFixListContains(hasDataReleaseTypeList, rel.r_type) + case _ => Bool(false) + } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadExclusive) } @@ -413,10 +420,10 @@ class MEICoherence extends CoherencePolicyWithUncached { )) } - def needsMemRead(a_type: UFix, global_state: UFix): Bool = { + def needsOuterRead(a_type: UFix, global_state: UFix): Bool = { (a_type != acquireWriteUncached) } - def needsMemWrite(a_type: UFix, global_state: UFix): Bool = { + def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } def needsAckReply(a_type: UFix, global_state: UFix): Bool = { @@ -534,9 +541,12 @@ class MSICoherence extends CoherencePolicyWithUncached { Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) } - def messageHasData (rel: Release): Bool = uFixListContains(hasDataReleaseTypeList, rel.r_type) - def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) - def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) + def messageHasData(msg: SourcedMessage) = msg match { + case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uFixListContains(hasDataReleaseTypeList, rel.r_type) + case _ => Bool(false) + } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } @@ -571,10 +581,10 @@ class MSICoherence extends CoherencePolicyWithUncached { )) } - def needsMemRead(a_type: UFix, global_state: UFix): Bool = { + def needsOuterRead(a_type: UFix, global_state: UFix): Bool = { (a_type != acquireWriteUncached) } - def needsMemWrite(a_type: UFix, global_state: UFix): Bool = { + def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } def needsAckReply(a_type: UFix, global_state: UFix): Bool = { @@ -693,9 +703,12 @@ class MESICoherence extends CoherencePolicyWithUncached { Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) } - def messageHasData (rel: Release): Bool = uFixListContains(hasDataReleaseTypeList, rel.r_type) - def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) - def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) + def messageHasData(msg: SourcedMessage) = msg match { + case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uFixListContains(hasDataReleaseTypeList, rel.r_type) + case _ => Bool(false) + } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } @@ -733,10 +746,10 @@ class MESICoherence extends CoherencePolicyWithUncached { )) } - def needsMemRead(a_type: UFix, global_state: UFix): Bool = { + def needsOuterRead(a_type: UFix, global_state: UFix): Bool = { (a_type != acquireWriteUncached) } - def needsMemWrite(a_type: UFix, global_state: UFix): Bool = { + def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } def needsAckReply(a_type: UFix, global_state: UFix): Bool = { @@ -871,9 +884,12 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) } - def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) - def messageHasData (rel: Release): Bool = uFixListContains(hasDataReleaseTypeList, rel.r_type) - def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) + def messageHasData(msg: SourcedMessage) = msg match { + case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uFixListContains(hasDataReleaseTypeList, rel.r_type) + case _ => Bool(false) + } def messageUpdatesDataArray (reply: Grant): Bool = { uFixListContains(List(grantReadShared, grantReadExclusive, grantReadMigratory), reply.g_type) } @@ -913,10 +929,10 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { )) } - def needsMemRead(a_type: UFix, global_state: UFix): Bool = { + def needsOuterRead(a_type: UFix, global_state: UFix): Bool = { (a_type != acquireWriteUncached && a_type != acquireInvalidateOthers) } - def needsMemWrite(a_type: UFix, global_state: UFix): Bool = { + def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached || a_type === acquireAtomicUncached) } def needsAckReply(a_type: UFix, global_state: UFix): Bool = { diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index b515c1c8..d348e0ff 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -11,7 +11,7 @@ abstract trait CoherenceConfigConstants { trait UncoreConstants { val NGLOBAL_XACTS = 8 - val MASTER_XACT_ID_BITS = log2Up(NGLOBAL_XACTS) + val MASTER_XACT_ID_MAX_BITS = log2Up(NGLOBAL_XACTS) val CACHE_DATA_SIZE_IN_BYTES = 1 << 6 } @@ -29,7 +29,7 @@ trait TileLinkTypeConstants { trait TileLinkSizeConstants extends TileLinkTypeConstants { - val CLIENT_XACT_ID_BITS = 5 + val CLIENT_XACT_ID_MAX_BITS = 10 val ACQUIRE_WRITE_MASK_BITS = 6 val ACQUIRE_SUBWORD_ADDR_BITS = 3 val ACQUIRE_ATOMIC_OP_BITS = 4 @@ -72,7 +72,7 @@ trait MemoryInterfaceConstants extends UncoreConstants with TileLinkSizeConstants { - val MEM_TAG_BITS = max(CLIENT_XACT_ID_BITS, MASTER_XACT_ID_BITS) + val MEM_TAG_BITS = max(CLIENT_XACT_ID_MAX_BITS, MASTER_XACT_ID_MAX_BITS) val MEM_DATA_BITS = 128 val REFILL_CYCLES = CACHE_DATA_SIZE_IN_BYTES*8/MEM_DATA_BITS } diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 7cce3ff3..492079cc 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -40,8 +40,6 @@ class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfi rdy := arb.ready && (in.bits.header.dst === UFix(i)) }} out <> rrarb.io.out - //out.bits.header.src := rrarb.io.chosen.toUFix - //out.bits.header.dst := UFix(i) }} for(i <- 0 until conf.nEndpoints) { io.in(i).ready := rdyVecs.map(r => r(i)).reduceLeft(_||_) @@ -62,21 +60,23 @@ class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { } object FIFOedLogicalNetworkIOWrapper { - def apply[T <: Data](in: FIFOIO[T])(implicit conf: LogicalNetworkConfiguration) = { - val shim = (new FIFOedLogicalNetworkIOWrapper){ in.bits.clone } + def apply[T <: Data](in: FIFOIO[T], src: UFix = UFix(0), dst: UFix = UFix(0))(implicit conf: LogicalNetworkConfiguration) = { + val shim = (new FIFOedLogicalNetworkIOWrapper(src, dst)){ in.bits.clone } shim.io.in.valid := in.valid shim.io.in.bits := in.bits in.ready := shim.io.in.ready shim.io.out } } -class FIFOedLogicalNetworkIOWrapper[T <: Data]()(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { +class FIFOedLogicalNetworkIOWrapper[T <: Data](src: UFix, dst: UFix)(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { val io = new Bundle { val in = (new FIFOIO){ data }.flip val out = (new FIFOIO){(new LogicalNetworkIO){ data }} } io.out.valid := io.in.valid io.out.bits.payload := io.in.bits + io.out.bits.header.dst := dst + io.out.bits.header.src := src io.in.ready := io.out.ready } diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index 73bf8aff..c16f8c9b 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -3,20 +3,30 @@ package uncore import Chisel._ import Constants._ -class PhysicalAddress extends Bundle { +trait HasPhysicalAddress extends Bundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) } -class MemData extends Bundle { +trait HasClientTransactionId extends Bundle { + val client_xact_id = Bits(width = CLIENT_XACT_ID_MAX_BITS) +} + +trait HasMasterTransactionId extends Bundle { + val master_xact_id = Bits(width = MASTER_XACT_ID_MAX_BITS) +} + +trait HasMemData extends Bundle { val data = Bits(width = MEM_DATA_BITS) } -class MemReqCmd extends PhysicalAddress { +class MemData extends Bundle with HasMemData + +class MemReqCmd extends Bundle with HasPhysicalAddress { val rw = Bool() val tag = Bits(width = MEM_TAG_BITS) } -class MemResp extends MemData { +class MemResp extends Bundle with HasMemData { val tag = Bits(width = MEM_TAG_BITS) } @@ -32,9 +42,12 @@ class ioMemPipe extends Bundle { val resp = (new PipeIO) { new MemResp() }.flip } -class Acquire extends PhysicalAddress { +trait SourcedMessage extends Bundle +trait ClientSourcedMessage extends SourcedMessage +trait MasterSourcedMessage extends SourcedMessage + +class Acquire extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId { val a_type = Bits(width = ACQUIRE_TYPE_MAX_BITS) - val client_xact_id = Bits(width = CLIENT_XACT_ID_BITS) val write_mask = Bits(width = ACQUIRE_WRITE_MASK_BITS) val subword_addr = Bits(width = ACQUIRE_SUBWORD_ADDR_BITS) val atomic_opcode = Bits(width = ACQUIRE_ATOMIC_OP_BITS) @@ -47,6 +60,9 @@ object Acquire acq.a_type := a_type acq.addr := addr acq.client_xact_id := client_xact_id + acq.write_mask := Bits(0, width = ACQUIRE_WRITE_MASK_BITS) + acq.subword_addr := Bits(0, width = ACQUIRE_SUBWORD_ADDR_BITS) + acq.atomic_opcode := Bits(0, width = ACQUIRE_ATOMIC_OP_BITS) acq } def apply(a_type: Bits, addr: UFix, client_xact_id: UFix, write_mask: Bits) = { @@ -55,6 +71,8 @@ object Acquire acq.addr := addr acq.client_xact_id := client_xact_id acq.write_mask := write_mask + acq.subword_addr := Bits(0, width = ACQUIRE_SUBWORD_ADDR_BITS) + acq.atomic_opcode := Bits(0, width = ACQUIRE_ATOMIC_OP_BITS) acq } def apply(a_type: Bits, addr: UFix, client_xact_id: UFix, subword_addr: UFix, atomic_opcode: UFix) = { @@ -64,15 +82,15 @@ object Acquire acq.client_xact_id := client_xact_id acq.subword_addr := subword_addr acq.atomic_opcode := atomic_opcode + acq.write_mask := Bits(0, width = ACQUIRE_WRITE_MASK_BITS) acq } } -class AcquireData extends MemData +class AcquireData extends ClientSourcedMessage with HasMemData -class Probe extends PhysicalAddress { +class Probe extends MasterSourcedMessage with HasPhysicalAddress with HasMasterTransactionId { val p_type = Bits(width = PROBE_TYPE_MAX_BITS) - val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) } object Release @@ -86,35 +104,105 @@ object Release rel } } -class Release extends PhysicalAddress { +class Release extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasMasterTransactionId { val r_type = Bits(width = RELEASE_TYPE_MAX_BITS) - val client_xact_id = Bits(width = CLIENT_XACT_ID_BITS) - val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) } -class ReleaseData extends MemData +class ReleaseData extends ClientSourcedMessage with HasMemData -class Grant extends MemData { +class Grant extends MasterSourcedMessage with HasMemData with HasClientTransactionId with HasMasterTransactionId { val g_type = Bits(width = GRANT_TYPE_MAX_BITS) - val client_xact_id = Bits(width = CLIENT_XACT_ID_BITS) - val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) } -class GrantAck extends Bundle { - val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) -} +class GrantAck extends ClientSourcedMessage with HasMasterTransactionId abstract class DirectionalFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) class ClientSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) class MasterSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) {flip()} -class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { +class UncachedTileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { val acquire = (new ClientSourcedIO){(new LogicalNetworkIO){new Acquire }} val acquire_data = (new ClientSourcedIO){(new LogicalNetworkIO){new AcquireData }} + val grant = (new MasterSourcedIO) {(new LogicalNetworkIO){new Grant }} + val grant_ack = (new ClientSourcedIO){(new LogicalNetworkIO){new GrantAck }} + override def clone = { new UncachedTileLinkIO().asInstanceOf[this.type] } +} + +class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends UncachedTileLinkIO()(conf) { val probe = (new MasterSourcedIO){(new LogicalNetworkIO){new Probe }} val release = (new ClientSourcedIO){(new LogicalNetworkIO){new Release }} val release_data = (new ClientSourcedIO){(new LogicalNetworkIO){new ReleaseData }} - val grant = (new MasterSourcedIO){(new LogicalNetworkIO){new Grant }} - val grant_ack = (new ClientSourcedIO){(new LogicalNetworkIO){new GrantAck }} override def clone = { new TileLinkIO().asInstanceOf[this.type] } } + +object UncachedTileLinkIOArbiterShim { + def apply[T <: HasClientTransactionId](in: ClientSourcedIO[LogicalNetworkIO[T]], id: Int, max: Int)(implicit lconf: LogicalNetworkConfiguration) = { + val shim = (new UncachedTileLinkIOArbiterShim(id, max)){in.bits.payload.clone} + shim.io.in <> in + shim.io.out + } +} +class UncachedTileLinkIOArbiterShim[T <: HasClientTransactionId](id: Int, max: Int)(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { + val io = new Bundle { + val in = (new ClientSourcedIO){(new LogicalNetworkIO){ data }}.flip + val out = (new ClientSourcedIO){(new LogicalNetworkIO){ data }} + } + io.out.bits := io.in.bits + io.out.bits.payload.client_xact_id := Cat(io.in.bits.payload.client_xact_id, UFix(id, log2Up(max))) + io.out.valid := io.in.valid + io.in.ready := io.out.ready +} + + +class UncachedTileLinkIOArbiter(n: Int)(implicit conf: LogicalNetworkConfiguration) extends Component { + val io = new Bundle { + val in = Vec(n) { new UncachedTileLinkIO }.flip + val out = new UncachedTileLinkIO + } + + val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) + val mem_cnt_next = mem_cnt + UFix(1) + val locked = Reg(resetVal = Bool(false)) + val lock_idx = Reg(resetVal = UFix(n)) + + when(io.out.acquire_data.valid && io.out.acquire_data.ready) { + mem_cnt := mem_cnt_next + when(!locked) { + locked := Bool(true) + lock_idx := Vec(io.in.map{ in => in.acquire_data.ready && in.acquire_data.valid}){Bool()}.indexWhere{i: Bool => i} + } + when(mem_cnt_next === UFix(0)) { + locked := Bool(false) + } + } + + val acqd_grant = ArbiterCtrl(io.in.map(_.acquire_data.valid)) + (0 until n).map(i => io.in(i).acquire_data.ready := Mux(locked, UFix(i) === lock_idx, acqd_grant(i)) && io.out.acquire_data.ready) + var acqd_bits = io.in(n-1).acquire_data.bits + for (i <- n-2 to 0 by -1) { + acqd_bits = Mux(io.in(i).acquire_data.valid, io.in(i).acquire_data.bits, acqd_bits) + } + val locked_req = io.in(lock_idx).acquire_data + io.out.acquire_data.bits := Mux(locked, locked_req.bits, acqd_bits) + io.out.acquire_data.valid := Mux(locked, locked_req.valid, io.in.map(_.acquire_data.valid).reduce(_||_)) + + val acq_arb = (new Arbiter(n)){ (new LogicalNetworkIO){new Acquire} } + io.out.acquire <> acq_arb.io.out + io.in.map(_.acquire).zipWithIndex.map{ case(acq, id) => UncachedTileLinkIOArbiterShim(acq, id, n) }.zip(acq_arb.io.in).map{ case (req, arb) => req <> arb} + + val grant_ack_arb = (new Arbiter(n)){ (new LogicalNetworkIO){new GrantAck} } + io.out.grant_ack <> grant_ack_arb.io.out + grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } + + io.out.grant.ready := Bool(false) + for (i <- 0 until n) { + val tag = io.out.grant.bits.payload.client_xact_id + io.in(i).grant.valid := Bool(false) + when (tag(log2Up(n)-1,0) === UFix(i)) { + io.in(i).grant.valid := io.out.grant.valid + io.out.grant.ready := io.in(i).grant.ready + } + io.in(i).grant.bits := io.out.grant.bits + io.in(i).grant.bits.payload.client_xact_id := tag >> UFix(log2Up(n)) + } +} diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 5609b701..14467f7b 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -3,688 +3,199 @@ package uncore import Chisel._ import Constants._ -class TrackerProbeData(implicit conf: CoherenceHubConfiguration) extends Bundle { +class TrackerProbeData(implicit conf: UncoreConfiguration) extends Bundle { val client_id = Bits(width = conf.ln.idBits) } -class TrackerAllocReq(implicit conf: CoherenceHubConfiguration) extends Bundle { +class TrackerAllocReq(implicit conf: UncoreConfiguration) extends Bundle { val acquire = new Acquire() val client_id = Bits(width = conf.ln.idBits) override def clone = { new TrackerAllocReq().asInstanceOf[this.type] } } class TrackerDependency extends Bundle { - val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) + val master_xact_id = Bits(width = MASTER_XACT_ID_MAX_BITS) } -class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) extends Component { - val co = conf.co - val io = new Bundle { - val alloc_req = (new FIFOIO) { new TrackerAllocReq }.flip - val p_data = (new PipeIO) { new TrackerProbeData }.flip - val can_alloc = Bool(INPUT) - val grant_ack = Bool(INPUT) - val release_cnt_dec = Bits(INPUT, conf.ln.nClients) - val probe_cnt_inc = Bits(INPUT, conf.ln.nClients) - val tile_incoherent = Bits(INPUT, conf.ln.nClients) - val release_data = (new PipeIO) { new ReleaseData }.flip - val acquire_data = (new PipeIO) { new AcquireData }.flip - val sent_grant_ack = Bool(INPUT) - val release_data_dep = (new PipeIO) { new TrackerDependency }.flip - val acquire_data_dep = (new PipeIO) { new TrackerDependency }.flip - - val mem_req_cmd = (new FIFOIO) { new MemReqCmd } - val mem_req_data = (new FIFOIO) { new MemData } - val mem_req_lock = Bool(OUTPUT) - val probe = (new FIFOIO) { new Probe } - val busy = Bool(OUTPUT) - val addr = Bits(OUTPUT, PADDR_BITS - OFFSET_BITS) - val init_client_id = Bits(OUTPUT, conf.ln.idBits) - val release_client_id = Bits(OUTPUT, conf.ln.idBits) - val client_xact_id = Bits(OUTPUT, CLIENT_XACT_ID_BITS) - val sharer_count = Bits(OUTPUT, conf.ln.idBits+1) - val a_type = Bits(OUTPUT, ACQUIRE_TYPE_MAX_BITS) - val push_probe = Bits(OUTPUT, conf.ln.nClients) - val pop_release = Bits(OUTPUT, conf.ln.nClients) - val pop_release_data = Bits(OUTPUT, conf.ln.nClients) - val pop_release_dep = Bits(OUTPUT, conf.ln.nClients) - val pop_acquire = Bits(OUTPUT, conf.ln.nClients) - val pop_acquire_data = Bits(OUTPUT, conf.ln.nClients) - val pop_acquire_dep = Bits(OUTPUT, conf.ln.nClients) - val send_grant_ack = Bool(OUTPUT) - } - - def doMemReqWrite(req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: PipeIO[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, client_id: UFix) { - req_cmd.bits.rw := Bool(true) - req_data.bits := data.bits - when(req_cmd.ready && req_cmd.valid) { - cmd_sent := Bool(true) - } - when (at_front_of_dep_queue) { - req_cmd.valid := !cmd_sent && req_data.ready && data.valid - lock := data.valid || cmd_sent - when (req_cmd.ready || cmd_sent) { - req_data.valid := data.valid - when(req_data.ready) { - pop_data := UFix(1) << client_id - when (data.valid) { - mem_cnt := mem_cnt_next - when(mem_cnt === UFix(REFILL_CYCLES-1)) { - pop_dep := UFix(1) << client_id - trigger := Bool(false) - } - } - } - } - } - } - - def doMemReqRead(req_cmd: FIFOIO[MemReqCmd], trigger: Bool) { - req_cmd.valid := Bool(true) - req_cmd.bits.rw := Bool(false) - when(req_cmd.ready) { - trigger := Bool(false) - } - } - - val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } - val state = Reg(resetVal = s_idle) - val xact = Reg{ new Acquire } - val init_client_id_ = Reg{ Bits() } - val release_count = if (conf.ln.nClients == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) - val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nClients)) - val release_client_id_ = Reg{ Bits() } - val x_needs_read = Reg(resetVal = Bool(false)) - val acquire_data_needs_write = Reg(resetVal = Bool(false)) - val release_data_needs_write = Reg(resetVal = Bool(false)) - val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) - val mem_cnt_next = mem_cnt + UFix(1) - val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) - val probe_initial_flags = Bits(width = conf.ln.nClients) - probe_initial_flags := Bits(0) - if (conf.ln.nClients > 1) { - val probe_self = co.needsSelfProbe(io.alloc_req.bits.acquire) - val probe_self_flag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.client_id(log2Up(conf.ln.nClients)-1,0))) - probe_initial_flags := ~(io.tile_incoherent | probe_self_flag) - } - - io.busy := state != s_idle - io.addr := xact.addr - io.init_client_id := init_client_id_ - io.release_client_id := release_client_id_ - io.client_xact_id := xact.client_xact_id - io.sharer_count := UFix(conf.ln.nClients) // TODO: Broadcast only - io.a_type := xact.a_type - - io.mem_req_cmd.valid := Bool(false) - io.mem_req_cmd.bits.rw := Bool(false) - io.mem_req_cmd.bits.addr := xact.addr - io.mem_req_cmd.bits.tag := UFix(id) - io.mem_req_data.valid := Bool(false) - io.mem_req_data.bits.data := UFix(0) - io.mem_req_lock := Bool(false) - io.probe.valid := Bool(false) - io.probe.bits.p_type := co.getProbeType(xact.a_type, UFix(0)) - io.probe.bits.master_xact_id := UFix(id) - io.probe.bits.addr := xact.addr - io.push_probe := Bits(0, width = conf.ln.nClients) - io.pop_release := Bits(0, width = conf.ln.nClients) - io.pop_release_data := Bits(0, width = conf.ln.nClients) - io.pop_release_dep := Bits(0, width = conf.ln.nClients) - io.pop_acquire := Bits(0, width = conf.ln.nClients) - io.pop_acquire_data := Bits(0, width = conf.ln.nClients) - io.pop_acquire_dep := Bits(0, width = conf.ln.nClients) - io.send_grant_ack := Bool(false) - - switch (state) { - is(s_idle) { - when( io.alloc_req.valid && io.can_alloc ) { - xact := io.alloc_req.bits.acquire - init_client_id_ := io.alloc_req.bits.client_id - acquire_data_needs_write := co.messageHasData(io.alloc_req.bits.acquire) - x_needs_read := co.needsMemRead(io.alloc_req.bits.acquire.a_type, UFix(0)) - probe_flags := probe_initial_flags - mem_cnt := UFix(0) - p_w_mem_cmd_sent := Bool(false) - x_w_mem_cmd_sent := Bool(false) - io.pop_acquire := UFix(1) << io.alloc_req.bits.client_id - if(conf.ln.nClients > 1) { - release_count := PopCount(probe_initial_flags) - state := Mux(probe_initial_flags.orR, s_probe, s_mem) - } else state := s_mem - } - } - is(s_probe) { - when(probe_flags.orR) { - io.push_probe := probe_flags - io.probe.valid := Bool(true) - } - when(io.probe_cnt_inc.orR) { - probe_flags := probe_flags & ~io.probe_cnt_inc // unflag sent reqs - } - when(io.release_cnt_dec.orR) { - val dec = PopCount(io.release_cnt_dec) - io.pop_release := io.release_cnt_dec - if(conf.ln.nClients > 1) release_count := release_count - dec - when(release_count === dec) { - state := s_mem - } - } - when(io.p_data.valid) { - release_data_needs_write := Bool(true) - release_client_id_ := io.p_data.bits.client_id - } - } - is(s_mem) { - when (release_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, - io.mem_req_data, - io.mem_req_lock, - io.release_data, - release_data_needs_write, - p_w_mem_cmd_sent, - io.pop_release_data, - io.pop_release_dep, - io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(id)), - release_client_id_) - } . elsewhen(acquire_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, - io.mem_req_data, - io.mem_req_lock, - io.acquire_data, - acquire_data_needs_write, - x_w_mem_cmd_sent, - io.pop_acquire_data, - io.pop_acquire_dep, - io.acquire_data_dep.valid && (io.acquire_data_dep.bits.master_xact_id === UFix(id)), - init_client_id_) - } . elsewhen (x_needs_read) { - doMemReqRead(io.mem_req_cmd, x_needs_read) - } . otherwise { - state := Mux(co.needsAckReply(xact.a_type, UFix(0)), s_ack, s_busy) - } - } - is(s_ack) { - io.send_grant_ack := Bool(true) - when(io.sent_grant_ack) { state := s_busy } - } - is(s_busy) { // Nothing left to do but wait for transaction to complete - when (io.grant_ack) { - state := s_idle - } - } - } -} - -case class CoherenceHubConfiguration(co: CoherencePolicy, ln: LogicalNetworkConfiguration) - -class CoherenceHubAdapter(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { - val io = new Bundle { - val net = (new TileLinkIO).flip - val hub = Vec(conf.nClients) { new TileLinkIO } - } - - val netClientProducedSubBundles = io.net.getClass.getMethods.filter( x => - classOf[ClientSourcedIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => - m.invoke(io.net).asInstanceOf[ClientSourcedIO[LogicalNetworkIO[Data]]] } - val netMasterProducedSubBundles = io.net.getClass.getMethods.filter( x => - classOf[MasterSourcedIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => - m.invoke(io.net).asInstanceOf[MasterSourcedIO[LogicalNetworkIO[Data]]] } - - val hubClientProducedSubBundles = io.hub.map{ io => { - io.getClass.getMethods.filter( x => - classOf[ClientSourcedIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => - m.invoke(io).asInstanceOf[ClientSourcedIO[LogicalNetworkIO[Data]]] }}}.transpose - val hubMasterProducedSubBundles = io.hub.map{ io => { - io.getClass.getMethods.filter( x => - classOf[MasterSourcedIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => - m.invoke(io).asInstanceOf[MasterSourcedIO[LogicalNetworkIO[Data]]] }}}.transpose - - hubMasterProducedSubBundles.zip(netMasterProducedSubBundles).foreach{ case(hub, net) => { - net.bits.header.src := UFix(0) - net.bits.header.dst := Vec(hub.map(_.valid)){Bool()}.indexWhere{s: Bool => s} - net.bits.payload := hub(0).bits.payload - net.valid := hub.map(_.valid).fold(Bool(false))(_||_) - hub.foreach( _.ready := net.ready) - }} - hubClientProducedSubBundles.zip(netClientProducedSubBundles).foreach{ case(hub, net) => { - hub.foreach(_.bits.header := net.bits.header) - hub.zipWithIndex.foreach{ case(h,i) => h.valid := (net.bits.header.src === UFix(i)) && net.valid } - hub.foreach(_.bits.payload := net.bits.payload) - net.ready := hub.map(_.ready).fold(Bool(false))(_||_) - }} -} - -abstract class CoherenceHub(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { - val io = new Bundle { - val tiles = Vec(conf.nClients) { new TileLinkIO }.flip - val incoherent = Vec(conf.nClients) { Bool() }.asInput - val mem = new ioMem - } -} - -class CoherenceHubNull(implicit conf: CoherenceHubConfiguration) extends CoherenceHub()(conf.ln) -{ - val co = conf.co.asInstanceOf[ThreeStateIncoherence] - - val acquire = io.tiles(0).acquire - val is_write = acquire.bits.payload.a_type === co.acquireWriteback - acquire.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp - io.mem.req_cmd.valid := acquire.valid && !(is_write && io.mem.resp.valid) - io.mem.req_cmd.bits.rw := is_write - io.mem.req_cmd.bits.tag := acquire.bits.payload.client_xact_id - io.mem.req_cmd.bits.addr := acquire.bits.payload.addr - io.mem.req_data <> io.tiles(0).acquire_data - - val grant = io.tiles(0).grant - grant.bits.payload.g_type := Mux(io.mem.resp.valid, co.grantData, co.grantAck) - grant.bits.payload.client_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, acquire.bits.payload.client_xact_id) - grant.bits.payload.master_xact_id := UFix(0) // don't care - grant.bits.payload.data := io.mem.resp.bits.data - grant.valid := io.mem.resp.valid || acquire.valid && is_write && io.mem.req_cmd.ready - - io.tiles(0).grant_ack.ready := Bool(true) - io.tiles(0).probe.valid := Bool(false) - io.tiles(0).release.ready := Bool(true) - io.tiles(0).release_data.ready := Bool(true) -} - - -class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends CoherenceHub()(conf.ln) -{ - implicit val lnConf = conf.ln - val co = conf.co - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTrackerBroadcast(_)) - - val busy_arr = Vec(NGLOBAL_XACTS){ Bool() } - val addr_arr = Vec(NGLOBAL_XACTS){ Bits(width=PADDR_BITS-OFFSET_BITS) } - val init_client_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } - val client_xact_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=CLIENT_XACT_ID_BITS) } - val a_type_arr = Vec(NGLOBAL_XACTS){ Bits(width=ACQUIRE_TYPE_MAX_BITS) } - val sh_count_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } - val send_grant_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } - - val do_free_arr = Vec(NGLOBAL_XACTS){ Bool() } - val release_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nClients){ Bool()} } - val probe_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nClients){ Bool()} } - val sent_grant_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } - val p_data_client_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } - val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } - - for( i <- 0 until NGLOBAL_XACTS) { - val t = trackerList(i).io - busy_arr(i) := t.busy - addr_arr(i) := t.addr - init_client_id_arr(i) := t.init_client_id - client_xact_id_arr(i) := t.client_xact_id - a_type_arr(i) := t.a_type - sh_count_arr(i) := t.sharer_count - send_grant_ack_arr(i) := t.send_grant_ack - t.grant_ack := do_free_arr(i) - t.p_data.bits.client_id := p_data_client_id_arr(i) - t.p_data.valid := p_data_valid_arr(i) - t.release_cnt_dec := release_cnt_dec_arr(i).toBits - t.probe_cnt_inc := probe_cnt_inc_arr(i).toBits - t.tile_incoherent := io.incoherent.toBits - t.sent_grant_ack := sent_grant_ack_arr(i) - do_free_arr(i) := Bool(false) - sent_grant_ack_arr(i) := Bool(false) - p_data_client_id_arr(i) := Bits(0, width = conf.ln.idBits) - p_data_valid_arr(i) := Bool(false) - for( j <- 0 until conf.ln.nClients) { - release_cnt_dec_arr(i)(j) := Bool(false) - probe_cnt_inc_arr(i)(j) := Bool(false) - } - } - - val release_data_dep_list = List.fill(conf.ln.nClients)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY - val acquire_data_dep_list = List.fill(conf.ln.nClients)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY - - // Free finished transactions - for( j <- 0 until conf.ln.nClients ) { - val ack = io.tiles(j).grant_ack - when (ack.valid) { - do_free_arr(ack.bits.payload.master_xact_id) := Bool(true) - } - ack.ready := Bool(true) - } - - // Reply to initial requestor - // Forward memory responses from mem to tile or arbitrate to ack - val mem_idx = io.mem.resp.bits.tag - val ack_idx = PriorityEncoder(send_grant_ack_arr.toBits) - for( j <- 0 until conf.ln.nClients ) { - val rep = io.tiles(j).grant - rep.bits.payload.g_type := UFix(0) - rep.bits.payload.client_xact_id := UFix(0) - rep.bits.payload.master_xact_id := UFix(0) - rep.bits.payload.data := io.mem.resp.bits.data - rep.valid := Bool(false) - when(io.mem.resp.valid && (UFix(j) === init_client_id_arr(mem_idx))) { - rep.bits.payload.g_type := co.getGrantType(a_type_arr(mem_idx), sh_count_arr(mem_idx)) - rep.bits.payload.client_xact_id := client_xact_id_arr(mem_idx) - rep.bits.payload.master_xact_id := mem_idx - rep.valid := Bool(true) - } . otherwise { - rep.bits.payload.g_type := co.getGrantType(a_type_arr(ack_idx), sh_count_arr(ack_idx)) - rep.bits.payload.client_xact_id := client_xact_id_arr(ack_idx) - rep.bits.payload.master_xact_id := ack_idx - when (UFix(j) === init_client_id_arr(ack_idx)) { - rep.valid := send_grant_ack_arr.toBits.orR - sent_grant_ack_arr(ack_idx) := rep.ready - } - } - } - io.mem.resp.ready := io.tiles(init_client_id_arr(mem_idx)).grant.ready - - // Create an arbiter for the one memory port - // We have to arbitrate between the different trackers' memory requests - // and once we have picked a request, get the right write data - val mem_req_cmd_arb = (new Arbiter(NGLOBAL_XACTS)) { new MemReqCmd() } - val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemData() } - for( i <- 0 until NGLOBAL_XACTS ) { - mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd - mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data - mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock - } - io.mem.req_cmd <> Queue(mem_req_cmd_arb.io.out) - io.mem.req_data <> Queue(mem_req_data_arb.io.out) - - // Handle probe replies, which may or may not have data - for( j <- 0 until conf.ln.nClients ) { - val release = io.tiles(j).release - val release_data = io.tiles(j).release_data - val idx = release.bits.payload.master_xact_id - val pop_releases = trackerList.map(_.io.pop_release(j).toBool) - val do_pop = foldR(pop_releases)(_ || _) - release.ready := Bool(true) - release_data_dep_list(j).io.enq.valid := release.valid && co.messageHasData(release.bits.payload) - release_data_dep_list(j).io.enq.bits.master_xact_id := release.bits.payload.master_xact_id - release_data.ready := foldR(trackerList.map(_.io.pop_release_data(j)))(_ || _) - when (release.valid && co.messageHasData(release.bits.payload)) { - p_data_valid_arr(idx) := Bool(true) - p_data_client_id_arr(idx) := UFix(j) - } - release_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_release_dep(j).toBool))(_||_) - } - for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.release_data.valid := io.tiles(trackerList(i).io.release_client_id).release_data.valid - trackerList(i).io.release_data.bits := io.tiles(trackerList(i).io.release_client_id).release_data.bits.payload - - trackerList(i).io.release_data_dep.valid := MuxLookup(trackerList(i).io.release_client_id, release_data_dep_list(0).io.deq.valid, (0 until conf.ln.nClients).map( j => UFix(j) -> release_data_dep_list(j).io.deq.valid)) - trackerList(i).io.release_data_dep.bits := MuxLookup(trackerList(i).io.release_client_id, release_data_dep_list(0).io.deq.bits, (0 until conf.ln.nClients).map( j => UFix(j) -> release_data_dep_list(j).io.deq.bits)) - - for( j <- 0 until conf.ln.nClients) { - val release = io.tiles(j).release - release_cnt_dec_arr(i)(j) := release.valid && (release.bits.payload.master_xact_id === UFix(i)) - } - } - - // Handle transaction initiation requests - // Only one allocation per cycle - // Init requests may or may not have data - val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } - val init_arb = (new Arbiter(conf.ln.nClients)) { new TrackerAllocReq } - for( i <- 0 until NGLOBAL_XACTS ) { - alloc_arb.io.in(i).valid := !trackerList(i).io.busy - trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready - trackerList(i).io.alloc_req.bits := init_arb.io.out.bits - trackerList(i).io.alloc_req.valid := init_arb.io.out.valid - - trackerList(i).io.acquire_data.bits := io.tiles(trackerList(i).io.init_client_id).acquire_data.bits.payload - trackerList(i).io.acquire_data.valid := io.tiles(trackerList(i).io.init_client_id).acquire_data.valid - trackerList(i).io.acquire_data_dep.bits := MuxLookup(trackerList(i).io.init_client_id, acquire_data_dep_list(0).io.deq.bits, (0 until conf.ln.nClients).map( j => UFix(j) -> acquire_data_dep_list(j).io.deq.bits)) - trackerList(i).io.acquire_data_dep.valid := MuxLookup(trackerList(i).io.init_client_id, acquire_data_dep_list(0).io.deq.valid, (0 until conf.ln.nClients).map( j => UFix(j) -> acquire_data_dep_list(j).io.deq.valid)) - } - for( j <- 0 until conf.ln.nClients ) { - val acquire = io.tiles(j).acquire - val acquire_data = io.tiles(j).acquire_data - val acquire_data_dep = acquire_data_dep_list(j).io.deq - init_arb.io.in(j).valid := acquire.valid - init_arb.io.in(j).bits.acquire := acquire.bits.payload - init_arb.io.in(j).bits.client_id := UFix(j) - val pop_acquires = trackerList.map(_.io.pop_acquire(j).toBool) - acquire.ready := foldR(pop_acquires)(_||_) - acquire_data_dep_list(j).io.enq.valid := acquire.ready && co.messageHasData(acquire.bits.payload) - acquire_data_dep_list(j).io.enq.bits.master_xact_id := OHToUFix(pop_acquires) - acquire_data.ready := foldR(trackerList.map(_.io.pop_acquire_data(j).toBool))(_||_) - acquire_data_dep.ready := foldR(trackerList.map(_.io.pop_acquire_dep(j).toBool))(_||_) - } - - alloc_arb.io.out.ready := init_arb.io.out.valid - - // Handle probe request generation - // Must arbitrate for each request port - val probe_arb_arr = List.fill(conf.ln.nClients)((new Arbiter(NGLOBAL_XACTS)) { new Probe() }) - for( j <- 0 until conf.ln.nClients ) { - for( i <- 0 until NGLOBAL_XACTS ) { - val t = trackerList(i).io - probe_arb_arr(j).io.in(i).bits := t.probe.bits - probe_arb_arr(j).io.in(i).valid := t.probe.valid && t.push_probe(j) - probe_cnt_inc_arr(i)(j) := probe_arb_arr(j).io.in(i).ready - } - FIFOedLogicalNetworkIOWrapper(probe_arb_arr(j).io.out) <> io.tiles(j).probe - } - -} +case class UncoreConfiguration(co: CoherencePolicyWithUncached, ln: LogicalNetworkConfiguration) abstract class CoherenceAgent(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { val io = new Bundle { - val network = (new TileLinkIO).flip + val client = (new TileLinkIO).flip + val master = new UncachedTileLinkIO val incoherent = Vec(conf.nClients) { Bool() }.asInput - val mem = new ioMem } } -class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends CoherenceAgent()(conf.ln) +class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends CoherenceAgent()(conf.ln) { implicit val lnConf = conf.ln val co = conf.co - val trackerList = new WritebackTracker(0) +: (1 to NGLOBAL_XACTS).map(new AcquireTracker(_)) + val trackerList = new WritebackTracker(0, bankId) +: (1 to NGLOBAL_XACTS).map(new AcquireTracker(_, bankId)) val release_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth must >= NPRIMARY val acquire_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth should >= NPRIMARY - for( i <- 0 to NGLOBAL_XACTS ) { - val t = trackerList(i) - t.io.tile_incoherent := io.incoherent.toBits - t.io.mem_resp.valid := io.mem.resp.valid && (io.mem.resp.bits.tag === UFix(i)) - t.io.mem_resp.bits := io.mem.resp.bits - } - io.mem.resp.ready := trackerList.map(_.io.mem_resp.ready).reduce(_||_) + trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) // Handle transaction initiation requests // Only one allocation per cycle // Init requests may or may not have data - val acquire = io.network.acquire - val acquire_data = io.network.acquire_data - val x_dep_deq = acquire_data_dep_q.io.deq + val acquire = io.client.acquire + val acquire_data = io.client.acquire_data + val acq_dep_deq = acquire_data_dep_q.io.deq val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val all_busy = trackerList.map(_.io.busy).reduce(_&&_) - val alloc_arb = (new Arbiter(NGLOBAL_XACTS+1)) { Bool() } - for( i <- 0 to NGLOBAL_XACTS ) { + val alloc_arb = (new Arbiter(trackerList.size)) { Bool() } + for( i <- 0 until trackerList.size ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy - trackerList(i).io.acquire.bits := acquire.bits - trackerList(i).io.acquire.valid := acquire.valid && alloc_arb.io.in(i).ready + val t = trackerList(i).io.client + t.acquire.bits := acquire.bits + t.acquire.valid := acquire.valid && alloc_arb.io.in(i).ready - trackerList(i).io.acquire_data.bits := acquire_data.bits - trackerList(i).io.acquire_data.valid := acquire_data.valid - trackerList(i).io.acquire_data_dep.bits := x_dep_deq.bits - trackerList(i).io.acquire_data_dep.valid := x_dep_deq.valid + t.acquire_data.bits := acquire_data.bits + t.acquire_data.valid := acquire_data.valid + trackerList(i).io.acquire_data_dep.bits := acq_dep_deq.bits + trackerList(i).io.acquire_data_dep.valid := acq_dep_deq.valid } - acquire.ready := trackerList.map(_.io.acquire.ready).reduce(_||_) - acquire_data.ready := trackerList.map(_.io.acquire_data.ready).reduce(_||_) + acquire.ready := trackerList.map(_.io.client.acquire.ready).reduce(_||_) + acquire_data.ready := trackerList.map(_.io.client.acquire_data.ready).reduce(_||_) acquire_data_dep_q.io.enq.valid := acquire.ready && co.messageHasData(acquire.bits.payload) - acquire_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(trackerList.map(_.io.acquire.ready)) - x_dep_deq.ready := trackerList.map(_.io.acquire_data_dep.ready).reduce(_||_) - + acquire_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(trackerList.map(_.io.client.acquire.ready)) + acq_dep_deq.ready := trackerList.map(_.io.acquire_data_dep.ready).reduce(_||_) alloc_arb.io.out.ready := acquire.valid // Handle probe request generation - val probe_arb = (new Arbiter(NGLOBAL_XACTS+1)){(new LogicalNetworkIO){ new Probe }} - for( i <- 0 to NGLOBAL_XACTS ) { - val t = trackerList(i).io - probe_arb.io.in(i).bits := t.probe.bits - probe_arb.io.in(i).valid := t.probe.valid - t.probe.ready := probe_arb.io.in(i).ready - } - io.network.probe <> probe_arb.io.out + val probe_arb = (new Arbiter(trackerList.size)){(new LogicalNetworkIO){ new Probe }} + io.client.probe <> probe_arb.io.out + probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.probe } // Handle probe replies, which may or may not have data - val release = io.network.release - val release_data = io.network.release_data + val release = io.client.release + val release_data = io.client.release_data val voluntary = co.isVoluntary(release.bits.payload) val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)){Bool()}.lastIndexWhere{b: Bool => b} val idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UFix(0)), release.bits.payload.master_xact_id) - release.ready := trackerList.map(_.io.release.ready).reduce(_||_) - release_data.ready := trackerList.map(_.io.release_data.ready).reduce(_||_) + release.ready := trackerList.map(_.io.client.release.ready).reduce(_||_) + release_data.ready := trackerList.map(_.io.client.release_data.ready).reduce(_||_) release_data_dep_q.io.enq.valid := release.valid && co.messageHasData(release.bits.payload) release_data_dep_q.io.enq.bits.master_xact_id := idx release_data_dep_q.io.deq.ready := trackerList.map(_.io.release_data_dep.ready).reduce(_||_) - for( i <- 0 to NGLOBAL_XACTS ) { - trackerList(i).io.release_data.valid := release_data.valid - trackerList(i).io.release_data.bits := release_data.bits + for( i <- 0 until trackerList.size ) { + val t = trackerList(i).io.client + t.release.valid := release.valid && (idx === UFix(i)) + t.release.bits := release.bits + t.release_data.valid := release_data.valid + t.release_data.bits := release_data.bits trackerList(i).io.release_data_dep.valid := release_data_dep_q.io.deq.valid trackerList(i).io.release_data_dep.bits := release_data_dep_q.io.deq.bits - trackerList(i).io.release.valid := release.valid && (idx === UFix(i)) - trackerList(i).io.release.bits := release.bits } // Reply to initial requestor - // Forward memory responses from mem to tile or arbitrate to ack - val grant_arb = (new Arbiter(NGLOBAL_XACTS+1)){(new LogicalNetworkIO){ new Grant }} - for( i <- 0 to NGLOBAL_XACTS ) { - val t = trackerList(i).io - grant_arb.io.in(i).bits := t.grant.bits - grant_arb.io.in(i).valid := t.grant.valid - t.grant.ready := grant_arb.io.in(i).ready - } - io.network.grant.valid := grant_arb.io.out.valid - io.network.grant.bits := grant_arb.io.out.bits - grant_arb.io.out.ready := io.network.grant.ready - when(io.mem.resp.valid) { - io.network.grant.valid := Bool(true) - io.network.grant.bits := Vec(trackerList.map(_.io.grant.bits)){(new LogicalNetworkIO){new Grant}}(io.mem.resp.bits.tag) - for( i <- 0 to NGLOBAL_XACTS ) { - trackerList(i).io.grant.ready := (io.mem.resp.bits.tag === UFix(i)) && io.network.grant.ready - } - } + val grant_arb = (new Arbiter(trackerList.size)){(new LogicalNetworkIO){ new Grant }} + io.client.grant <> grant_arb.io.out + grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.grant } // Free finished transactions - val ack = io.network.grant_ack - for( i <- 0 to NGLOBAL_XACTS ) { - trackerList(i).io.free := ack.valid && (ack.bits.payload.master_xact_id === UFix(i)) - } + val ack = io.client.grant_ack + trackerList.map(_.io.client.grant_ack.valid := ack.valid) + trackerList.map(_.io.client.grant_ack.bits := ack.bits) ack.ready := Bool(true) // Create an arbiter for the one memory port - // We have to arbitrate between the different trackers' memory requests - // and once we have picked a request, get the right write data - val mem_req_cmd_arb = (new Arbiter(NGLOBAL_XACTS+1)) { new MemReqCmd() } - val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS+1)) { new MemData() } - for( i <- 0 to NGLOBAL_XACTS ) { - mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd - mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data - mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock - } - io.mem.req_cmd <> Queue(mem_req_cmd_arb.io.out) - io.mem.req_data <> Queue(mem_req_data_arb.io.out) + val outer_arb = new UncachedTileLinkIOArbiter(trackerList.size) + outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.master } + io.master <> outer_arb.io.out } -abstract class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Component with MemoryRequestGenerator { +abstract class XactTracker()(implicit conf: UncoreConfiguration) extends Component with OuterRequestGenerator { val co = conf.co implicit val ln = conf.ln val io = new Bundle { - val acquire = (new FIFOIO){(new LogicalNetworkIO) { new Acquire }}.flip - val acquire_data = (new FIFOIO){(new LogicalNetworkIO) { new AcquireData }}.flip - val release = (new FIFOIO){(new LogicalNetworkIO) { new Release }}.flip - val release_data = (new FIFOIO){(new LogicalNetworkIO) { new ReleaseData }}.flip - val free = Bool(INPUT) - val tile_incoherent = Bits(INPUT, conf.ln.nClients) - val release_data_dep = (new FIFOIO) { new TrackerDependency }.flip - val acquire_data_dep = (new FIFOIO) { new TrackerDependency }.flip - val mem_resp = (new FIFOIO) { new MemResp }.flip + val client = (new TileLinkIO).flip + val master = new UncachedTileLinkIO - val mem_req_cmd = (new FIFOIO) { new MemReqCmd } - val mem_req_data = (new FIFOIO) { new MemData } - val mem_req_lock = Bool(OUTPUT) - val probe = (new FIFOIO) {(new LogicalNetworkIO) { new Probe }} - val grant = (new FIFOIO) {(new LogicalNetworkIO) { new Grant }} - val busy = Bool(OUTPUT) - val has_acquire_conflict = Bool(OUTPUT) - val has_release_conflict = Bool(OUTPUT) + val tile_incoherent = Bits(INPUT, conf.ln.nClients) + val release_data_dep = (new FIFOIO) { new TrackerDependency }.flip + val acquire_data_dep = (new FIFOIO) { new TrackerDependency }.flip + + val busy = Bool(OUTPUT) + val has_acquire_conflict = Bool(OUTPUT) + val has_release_conflict = Bool(OUTPUT) } } -class WritebackTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends XactTracker(id)(conf) { +class WritebackTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfiguration) extends XactTracker()(conf) { val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(4){ UFix() } val state = Reg(resetVal = s_idle) val xact = Reg{ new Release } val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) val release_data_needs_write = Reg(resetVal = Bool(false)) val mem_cmd_sent = Reg(resetVal = Bool(false)) + val cmd_to_write = co.getUncachedWriteAcquire(xact.addr, UFix(trackerId)) + val cmd_to_read = co.getUncachedReadAcquire(xact.addr, UFix(trackerId)) - io.acquire.ready := Bool(false) - io.acquire_data.ready := Bool(false) io.acquire_data_dep.ready := Bool(false) - io.mem_resp.ready := Bool(false) - io.probe.valid := Bool(false) + io.release_data_dep.ready := Bool(false) io.busy := Bool(true) io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.release.bits.payload.addr) && (state != s_idle) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.bits.payload.addr) && (state != s_idle) - io.mem_req_cmd.valid := Bool(false) - io.mem_req_cmd.bits.rw := Bool(false) - io.mem_req_cmd.bits.addr := xact.addr - io.mem_req_cmd.bits.tag := UFix(id) - io.mem_req_data.valid := Bool(false) - io.mem_req_data.bits.data := UFix(0) - io.mem_req_lock := Bool(false) - io.release.ready := Bool(false) - io.release_data.ready := Bool(false) - io.release_data_dep.ready := Bool(false) - io.grant.valid := Bool(false) - io.grant.bits.payload.g_type := co.getGrantType(xact, UFix(0)) - io.grant.bits.payload.client_xact_id := xact.client_xact_id - io.grant.bits.payload.master_xact_id := UFix(id) - io.grant.bits.header.dst := init_client_id_ + io.master.grant.ready := Bool(false) + io.master.acquire.valid := Bool(false) + io.master.acquire.bits.payload := cmd_to_write + //TODO io.master.acquire.bits.header.dst + io.master.acquire.bits.header.src := UFix(bankId) + io.master.acquire_data.valid := Bool(false) + io.master.acquire_data.bits.payload.data := UFix(0) + //TODO io.master.acquire_data.bits.header.dst + io.master.acquire_data.bits.header.src := UFix(bankId) + io.client.acquire.ready := Bool(false) + io.client.acquire_data.ready := Bool(false) + io.client.probe.valid := Bool(false) + io.client.release.ready := Bool(false) + io.client.release_data.ready := Bool(false) + io.client.grant.valid := Bool(false) + io.client.grant.bits.payload.g_type := co.getGrantType(xact, UFix(0)) + io.client.grant.bits.payload.client_xact_id := xact.client_xact_id + io.client.grant.bits.payload.master_xact_id := UFix(trackerId) + io.client.grant.bits.header.dst := init_client_id_ + io.client.grant.bits.header.src := UFix(bankId) + io.client.grant_ack.valid := Bool(false) switch (state) { is(s_idle) { - when( io.release.valid ) { - xact := io.release.bits.payload - init_client_id_ := io.release.bits.header.src - release_data_needs_write := co.messageHasData(io.release.bits.payload) + when( io.client.release.valid ) { + xact := io.client.release.bits.payload + init_client_id_ := io.client.release.bits.header.src + release_data_needs_write := co.messageHasData(io.client.release.bits.payload) mem_cnt := UFix(0) mem_cmd_sent := Bool(false) - io.release.ready := Bool(true) + io.client.release.ready := Bool(true) state := s_mem } } is(s_mem) { when (release_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, - io.mem_req_data, - io.mem_req_lock, - io.release_data, + doOuterReqWrite(io.master.acquire, + io.master.acquire_data, + io.client.release_data, release_data_needs_write, mem_cmd_sent, io.release_data_dep.ready, - io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(id))) + io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(trackerId))) } . otherwise { state := s_ack } } is(s_ack) { - io.grant.valid := Bool(true) - when(io.grant.ready) { state := s_idle } + io.client.grant.valid := Bool(true) + when(io.client.grant.ready) { state := s_idle } } } } -class AcquireTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends XactTracker(id)(conf) { +class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfiguration) extends XactTracker()(conf) { val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) val xact = Reg{ new Acquire } @@ -697,63 +208,65 @@ class AcquireTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends val x_needs_read = Reg(resetVal = Bool(false)) val acquire_data_needs_write = Reg(resetVal = Bool(false)) val release_data_needs_write = Reg(resetVal = Bool(false)) - val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val cmd_to_write = co.getUncachedWriteAcquire(xact.addr, UFix(trackerId)) + val cmd_to_read = co.getUncachedReadAcquire(xact.addr, UFix(trackerId)) + val a_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val r_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val probe_initial_flags = Bits(width = conf.ln.nClients) probe_initial_flags := Bits(0) if (conf.ln.nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence - // TODO: this is hackish; figure out how to do it more systematically - val probe_self = co match { - case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.acquire.bits.payload) - case _ => Bool(false) - } - val myflag = Mux(probe_self, Bits(0), UFixToOH(io.acquire.bits.header.src(log2Up(conf.ln.nClients)-1,0))) + val probe_self = co.needsSelfProbe(io.client.acquire.bits.payload) + val myflag = Mux(probe_self, Bits(0), UFixToOH(io.client.acquire.bits.header.src(log2Up(conf.ln.nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } io.busy := state != s_idle - io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, io.acquire.bits.payload.addr) && (state != s_idle) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.release.bits.payload.addr) && (state != s_idle) - io.mem_req_cmd.valid := Bool(false) - io.mem_req_cmd.bits.rw := Bool(false) - io.mem_req_cmd.bits.addr := xact.addr - io.mem_req_cmd.bits.tag := UFix(id) - io.mem_req_data.valid := Bool(false) - io.mem_req_data.bits.data := UFix(0) - io.mem_req_lock := Bool(false) - io.probe.valid := Bool(false) - io.probe.bits.payload.p_type := co.getProbeType(xact.a_type, UFix(0)) - io.probe.bits.payload.master_xact_id := UFix(id) - io.probe.bits.payload.addr := xact.addr - io.probe.bits.header.dst := UFix(0) - io.grant.bits.payload.data := io.mem_resp.bits.data - io.grant.bits.payload.g_type := grant_type - io.grant.bits.payload.client_xact_id := xact.client_xact_id - io.grant.bits.payload.master_xact_id := UFix(id) - io.grant.bits.header.dst := init_client_id_ - io.grant.valid := (io.mem_resp.valid && (UFix(id) === io.mem_resp.bits.tag)) - io.acquire.ready := Bool(false) - io.acquire_data.ready := Bool(false) + io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, io.client.acquire.bits.payload.addr) && (state != s_idle) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.bits.payload.addr) && (state != s_idle) + io.master.acquire.valid := Bool(false) + io.master.acquire.bits.payload := co.getUncachedReadAcquire(xact.addr, UFix(trackerId)) + //TODO io.master.acquire.bits.header.dst + io.master.acquire.bits.header.src := UFix(bankId) + io.master.acquire_data.valid := Bool(false) + io.master.acquire_data.bits.payload.data := UFix(0) + //TODO io.master.acquire_data.bits.header.dst + io.master.acquire_data.bits.header := UFix(bankId) + io.client.probe.valid := Bool(false) + io.client.probe.bits.payload.p_type := co.getProbeType(xact.a_type, UFix(0)) + io.client.probe.bits.payload.master_xact_id := UFix(trackerId) + io.client.probe.bits.payload.addr := xact.addr + io.client.probe.bits.header.dst := UFix(0) + io.client.probe.bits.header.src := UFix(bankId) + io.client.grant.bits.payload.data := io.master.grant.bits.payload.data + io.client.grant.bits.payload.g_type := grant_type + io.client.grant.bits.payload.client_xact_id := xact.client_xact_id + io.client.grant.bits.payload.master_xact_id := UFix(trackerId) + io.client.grant.bits.header.dst := init_client_id_ + io.client.grant.bits.header.src := UFix(bankId) + io.client.grant.valid := (io.master.grant.valid && (UFix(trackerId) === io.master.grant.bits.payload.client_xact_id)) + io.client.acquire.ready := Bool(false) + io.client.acquire_data.ready := Bool(false) io.acquire_data_dep.ready := Bool(false) - io.release.ready := Bool(false) - io.release_data.ready := Bool(false) + io.client.release.ready := Bool(false) + io.client.release_data.ready := Bool(false) io.release_data_dep.ready := Bool(false) - io.mem_resp.ready := io.grant.ready + io.master.grant.ready := io.client.grant.ready + io.client.grant_ack.valid := Bool(false) switch (state) { is(s_idle) { - when( io.acquire.valid ) { - xact := io.acquire.bits.payload - init_client_id_ := io.acquire.bits.header.src + when( io.client.acquire.valid ) { + xact := io.client.acquire.bits.payload + init_client_id_ := io.client.acquire.bits.header.src init_sharer_cnt_ := UFix(conf.ln.nClients) // TODO: Broadcast only - acquire_data_needs_write := co.messageHasData(io.acquire.bits.payload) - x_needs_read := co.needsMemRead(io.acquire.bits.payload.a_type, UFix(0)) + acquire_data_needs_write := co.messageHasData(io.client.acquire.bits.payload) + x_needs_read := co.needsOuterRead(io.client.acquire.bits.payload.a_type, UFix(0)) probe_flags := probe_initial_flags mem_cnt := UFix(0) - p_w_mem_cmd_sent := Bool(false) - x_w_mem_cmd_sent := Bool(false) - io.acquire.ready := Bool(true) + r_w_mem_cmd_sent := Bool(false) + a_w_mem_cmd_sent := Bool(false) + io.client.acquire.ready := Bool(true) if(conf.ln.nClients > 1) { release_count := PopCount(probe_initial_flags) state := Mux(probe_initial_flags.orR, s_probe, s_mem) @@ -763,78 +276,76 @@ class AcquireTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends is(s_probe) { val curr_p_id = PriorityEncoder(probe_flags) when(probe_flags.orR) { - io.probe.valid := Bool(true) - io.probe.bits.header.dst := curr_p_id + io.client.probe.valid := Bool(true) + io.client.probe.bits.header.dst := curr_p_id } - when(io.probe.ready) { + when(io.client.probe.ready) { probe_flags := probe_flags & ~(UFixToOH(curr_p_id)) } - when(io.release.valid) { - io.release.ready := Bool(true) + when(io.client.release.valid) { + io.client.release.ready := Bool(true) if(conf.ln.nClients > 1) release_count := release_count - UFix(1) when(release_count === UFix(1)) { state := s_mem } - release_data_needs_write := co.messageHasData(io.release.bits.payload) + release_data_needs_write := co.messageHasData(io.client.release.bits.payload) } } is(s_mem) { when (release_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, - io.mem_req_data, - io.mem_req_lock, - io.release_data, + doOuterReqWrite(io.master.acquire, + io.master.acquire_data, + io.client.release_data, release_data_needs_write, - p_w_mem_cmd_sent, + r_w_mem_cmd_sent, io.release_data_dep.ready, - io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(id))) + io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(trackerId))) } . elsewhen(acquire_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, - io.mem_req_data, - io.mem_req_lock, - io.acquire_data, + doOuterReqWrite(io.master.acquire, + io.master.acquire_data, + io.client.acquire_data, acquire_data_needs_write, - x_w_mem_cmd_sent, + a_w_mem_cmd_sent, io.acquire_data_dep.ready, - io.acquire_data_dep.valid && (io.acquire_data_dep.bits.master_xact_id === UFix(id))) + io.acquire_data_dep.valid && (io.acquire_data_dep.bits.master_xact_id === UFix(trackerId))) } . elsewhen (x_needs_read) { - doMemReqRead(io.mem_req_cmd, x_needs_read) + doOuterReqRead(io.master.acquire, x_needs_read) } . otherwise { state := Mux(co.needsAckReply(xact.a_type, UFix(0)), s_ack, - Mux(co.requiresAck(io.grant.bits.payload), s_busy, s_idle)) + Mux(co.requiresAck(io.client.grant.bits.payload), s_busy, s_idle)) } } is(s_ack) { - io.grant.valid := Bool(true) - when(io.grant.ready) { state := Mux(co.requiresAck(io.grant.bits.payload), s_busy, s_idle) } + io.client.grant.valid := Bool(true) + when(io.client.grant.ready) { state := Mux(co.requiresAck(io.client.grant.bits.payload), s_busy, s_idle) } } is(s_busy) { // Nothing left to do but wait for transaction to complete - when (io.free) { + when (io.client.grant_ack.valid && io.client.grant_ack.bits.payload.master_xact_id === UFix(trackerId)) { state := s_idle } } } } -trait MemoryRequestGenerator { +abstract trait OuterRequestGenerator { + val cmd_to_write: Acquire + val cmd_to_read: Acquire val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) - val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) - def doMemReqWrite[T <: Data](req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: FIFOIO[LogicalNetworkIO[T]], trigger: Bool, cmd_sent: Bool, pop_dep: Bool, at_front_of_dep_queue: Bool) { - req_cmd.bits.rw := Bool(true) - req_data.bits := data.bits.payload - when(req_cmd.ready && req_cmd.valid) { + def doOuterReqWrite[T <: Data](master_acq: FIFOIO[LogicalNetworkIO[Acquire]], master_acq_data: FIFOIO[LogicalNetworkIO[AcquireData]], client_data: FIFOIO[LogicalNetworkIO[T]], trigger: Bool, cmd_sent: Bool, pop_dep: Bool, at_front_of_dep_queue: Bool) { + master_acq.bits.payload := cmd_to_write + master_acq_data.bits.payload := client_data.bits.payload + when(master_acq.ready && master_acq.valid) { cmd_sent := Bool(true) } when (at_front_of_dep_queue) { - req_cmd.valid := !cmd_sent && req_data.ready && data.valid - lock := data.valid || cmd_sent - when (req_cmd.ready || cmd_sent) { - req_data.valid := data.valid - when(req_data.ready) { - data.ready:= Bool(true) - when (data.valid) { + master_acq.valid := !cmd_sent && master_acq_data.ready && client_data.valid + when (master_acq.ready || cmd_sent) { + master_acq_data.valid := client_data.valid + when(master_acq_data.ready) { + client_data.ready:= Bool(true) + when (client_data.valid) { mem_cnt := mem_cnt_next when(mem_cnt === UFix(REFILL_CYCLES-1)) { pop_dep := Bool(true) @@ -846,10 +357,10 @@ trait MemoryRequestGenerator { } } - def doMemReqRead(req_cmd: FIFOIO[MemReqCmd], trigger: Bool) { - req_cmd.valid := Bool(true) - req_cmd.bits.rw := Bool(false) - when(req_cmd.ready) { + def doOuterReqRead(master_acq: FIFOIO[LogicalNetworkIO[Acquire]], trigger: Bool) { + master_acq.valid := Bool(true) + master_acq.bits.payload := cmd_to_read + when(master_acq.ready) { trigger := Bool(false) } } From 4d007d5c404168104d23473d09615c8aee7930fd Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 22 Jan 2013 20:09:21 -0800 Subject: [PATCH 169/374] changed val names in hub to match new tilelink names --- uncore/src/uncore.scala | 460 ++++++++++++++++++++-------------------- 1 file changed, 234 insertions(+), 226 deletions(-) diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 01e9c40c..9d736e8b 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -24,14 +24,14 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val p_data = (new PipeIO) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) val grant_ack = Bool(INPUT) - val p_rep_cnt_dec = Bits(INPUT, conf.ln.nTiles) - val p_req_cnt_inc = Bits(INPUT, conf.ln.nTiles) + val release_cnt_dec = Bits(INPUT, conf.ln.nTiles) + val probe_cnt_inc = Bits(INPUT, conf.ln.nTiles) val tile_incoherent = Bits(INPUT, conf.ln.nTiles) - val p_rep_data = (new PipeIO) { new ReleaseData }.flip - val x_init_data = (new PipeIO) { new AcquireData }.flip + val release_data = (new PipeIO) { new ReleaseData }.flip + val acquire_data = (new PipeIO) { new AcquireData }.flip val sent_grant_ack = Bool(INPUT) - val p_rep_data_dep = (new PipeIO) { new TrackerDependency }.flip - val x_init_data_dep = (new PipeIO) { new TrackerDependency }.flip + val release_data_dep = (new PipeIO) { new TrackerDependency }.flip + val acquire_data_dep = (new PipeIO) { new TrackerDependency }.flip val mem_req_cmd = (new FIFOIO) { new MemReqCmd } val mem_req_data = (new FIFOIO) { new MemData } @@ -40,17 +40,17 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val busy = Bool(OUTPUT) val addr = Bits(OUTPUT, PADDR_BITS - OFFSET_BITS) val init_client_id = Bits(OUTPUT, conf.ln.idBits) - val p_rep_client_id = Bits(OUTPUT, conf.ln.idBits) + val release_client_id = Bits(OUTPUT, conf.ln.idBits) val client_xact_id = Bits(OUTPUT, CLIENT_XACT_ID_BITS) val sharer_count = Bits(OUTPUT, conf.ln.idBits+1) val a_type = Bits(OUTPUT, ACQUIRE_TYPE_MAX_BITS) - val push_p_req = Bits(OUTPUT, conf.ln.nTiles) - val pop_p_rep = Bits(OUTPUT, conf.ln.nTiles) - val pop_p_rep_data = Bits(OUTPUT, conf.ln.nTiles) - val pop_p_rep_dep = Bits(OUTPUT, conf.ln.nTiles) - val pop_x_init = Bits(OUTPUT, conf.ln.nTiles) - val pop_x_init_data = Bits(OUTPUT, conf.ln.nTiles) - val pop_x_init_dep = Bits(OUTPUT, conf.ln.nTiles) + val push_probe = Bits(OUTPUT, conf.ln.nTiles) + val pop_release = Bits(OUTPUT, conf.ln.nTiles) + val pop_release_data = Bits(OUTPUT, conf.ln.nTiles) + val pop_release_dep = Bits(OUTPUT, conf.ln.nTiles) + val pop_acquire = Bits(OUTPUT, conf.ln.nTiles) + val pop_acquire_data = Bits(OUTPUT, conf.ln.nTiles) + val pop_acquire_dep = Bits(OUTPUT, conf.ln.nTiles) val send_grant_ack = Bool(OUTPUT) } @@ -91,19 +91,19 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val state = Reg(resetVal = s_idle) val xact = Reg{ new Acquire } val init_client_id_ = Reg{ Bits() } - val p_rep_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) - val p_req_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) - val p_rep_client_id_ = Reg{ Bits() } + val release_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) + val release_client_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) - val x_init_data_needs_write = Reg(resetVal = Bool(false)) - val p_rep_data_needs_write = Reg(resetVal = Bool(false)) + val acquire_data_needs_write = Reg(resetVal = Bool(false)) + val release_data_needs_write = Reg(resetVal = Bool(false)) val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) - val p_req_initial_flags = Bits(width = conf.ln.nTiles) - p_req_initial_flags := Bits(0) + val probe_initial_flags = Bits(width = conf.ln.nTiles) + probe_initial_flags := Bits(0) if (conf.ln.nTiles > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence // TODO: this is hackish; figure out how to do it more systematically @@ -112,13 +112,13 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex case _ => Bool(false) } val myflag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.client_id(log2Up(conf.ln.nTiles)-1,0))) - p_req_initial_flags := ~(io.tile_incoherent | myflag) + probe_initial_flags := ~(io.tile_incoherent | myflag) } io.busy := state != s_idle io.addr := xact.addr io.init_client_id := init_client_id_ - io.p_rep_client_id := p_rep_client_id_ + io.release_client_id := release_client_id_ io.client_xact_id := xact.client_xact_id io.sharer_count := UFix(conf.ln.nTiles) // TODO: Broadcast only io.a_type := xact.a_type @@ -134,13 +134,13 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex io.probe.bits.p_type := co.getProbeType(xact.a_type, UFix(0)) io.probe.bits.master_xact_id := UFix(id) io.probe.bits.addr := xact.addr - io.push_p_req := Bits(0, width = conf.ln.nTiles) - io.pop_p_rep := Bits(0, width = conf.ln.nTiles) - io.pop_p_rep_data := Bits(0, width = conf.ln.nTiles) - io.pop_p_rep_dep := Bits(0, width = conf.ln.nTiles) - io.pop_x_init := Bits(0, width = conf.ln.nTiles) - io.pop_x_init_data := Bits(0, width = conf.ln.nTiles) - io.pop_x_init_dep := Bits(0, width = conf.ln.nTiles) + io.push_probe := Bits(0, width = conf.ln.nTiles) + io.pop_release := Bits(0, width = conf.ln.nTiles) + io.pop_release_data := Bits(0, width = conf.ln.nTiles) + io.pop_release_dep := Bits(0, width = conf.ln.nTiles) + io.pop_acquire := Bits(0, width = conf.ln.nTiles) + io.pop_acquire_data := Bits(0, width = conf.ln.nTiles) + io.pop_acquire_dep := Bits(0, width = conf.ln.nTiles) io.send_grant_ack := Bool(false) switch (state) { @@ -148,62 +148,62 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex when( io.alloc_req.valid && io.can_alloc ) { xact := io.alloc_req.bits.acquire init_client_id_ := io.alloc_req.bits.client_id - x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.acquire) + acquire_data_needs_write := co.messageHasData(io.alloc_req.bits.acquire) x_needs_read := co.needsMemRead(io.alloc_req.bits.acquire.a_type, UFix(0)) - p_req_flags := p_req_initial_flags + probe_flags := probe_initial_flags mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) - io.pop_x_init := UFix(1) << io.alloc_req.bits.client_id + io.pop_acquire := UFix(1) << io.alloc_req.bits.client_id if(conf.ln.nTiles > 1) { - p_rep_count := PopCount(p_req_initial_flags) - state := Mux(p_req_initial_flags.orR, s_probe, s_mem) + release_count := PopCount(probe_initial_flags) + state := Mux(probe_initial_flags.orR, s_probe, s_mem) } else state := s_mem } } is(s_probe) { - when(p_req_flags.orR) { - io.push_p_req := p_req_flags + when(probe_flags.orR) { + io.push_probe := probe_flags io.probe.valid := Bool(true) } - when(io.p_req_cnt_inc.orR) { - p_req_flags := p_req_flags & ~io.p_req_cnt_inc // unflag sent reqs + when(io.probe_cnt_inc.orR) { + probe_flags := probe_flags & ~io.probe_cnt_inc // unflag sent reqs } - when(io.p_rep_cnt_dec.orR) { - val dec = PopCount(io.p_rep_cnt_dec) - io.pop_p_rep := io.p_rep_cnt_dec - if(conf.ln.nTiles > 1) p_rep_count := p_rep_count - dec - when(p_rep_count === dec) { + when(io.release_cnt_dec.orR) { + val dec = PopCount(io.release_cnt_dec) + io.pop_release := io.release_cnt_dec + if(conf.ln.nTiles > 1) release_count := release_count - dec + when(release_count === dec) { state := s_mem } } when(io.p_data.valid) { - p_rep_data_needs_write := Bool(true) - p_rep_client_id_ := io.p_data.bits.client_id + release_data_needs_write := Bool(true) + release_client_id_ := io.p_data.bits.client_id } } is(s_mem) { - when (p_rep_data_needs_write) { + when (release_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, - io.p_rep_data, - p_rep_data_needs_write, + io.release_data, + release_data_needs_write, p_w_mem_cmd_sent, - io.pop_p_rep_data, - io.pop_p_rep_dep, - io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.master_xact_id === UFix(id)), - p_rep_client_id_) - } . elsewhen(x_init_data_needs_write) { + io.pop_release_data, + io.pop_release_dep, + io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(id)), + release_client_id_) + } . elsewhen(acquire_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, - io.x_init_data, - x_init_data_needs_write, + io.acquire_data, + acquire_data_needs_write, x_w_mem_cmd_sent, - io.pop_x_init_data, - io.pop_x_init_dep, - io.x_init_data_dep.valid && (io.x_init_data_dep.bits.master_xact_id === UFix(id)), + io.pop_acquire_data, + io.pop_acquire_dep, + io.acquire_data_dep.valid && (io.acquire_data_dep.bits.master_xact_id === UFix(id)), init_client_id_) } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) @@ -274,22 +274,22 @@ class CoherenceHubNull(implicit conf: CoherenceHubConfiguration) extends Coheren { val co = conf.co.asInstanceOf[ThreeStateIncoherence] - val x_init = io.tiles(0).acquire - val is_write = x_init.bits.payload.a_type === co.acquireWriteback - x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp - io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) + val acquire = io.tiles(0).acquire + val is_write = acquire.bits.payload.a_type === co.acquireWriteback + acquire.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp + io.mem.req_cmd.valid := acquire.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write - io.mem.req_cmd.bits.tag := x_init.bits.payload.client_xact_id - io.mem.req_cmd.bits.addr := x_init.bits.payload.addr + io.mem.req_cmd.bits.tag := acquire.bits.payload.client_xact_id + io.mem.req_cmd.bits.addr := acquire.bits.payload.addr io.mem.req_data <> io.tiles(0).acquire_data val grant = io.tiles(0).grant grant.bits.payload.g_type := Mux(io.mem.resp.valid, co.grantData, co.grantAck) - grant.bits.payload.client_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.payload.client_xact_id) + grant.bits.payload.client_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, acquire.bits.payload.client_xact_id) grant.bits.payload.master_xact_id := UFix(0) // don't care grant.bits.payload.data := io.mem.resp.bits.data grant.bits.payload.require_ack := Bool(true) - grant.valid := io.mem.resp.valid || x_init.valid && is_write && io.mem.req_cmd.ready + grant.valid := io.mem.resp.valid || acquire.valid && is_write && io.mem.req_cmd.ready io.tiles(0).abort.valid := Bool(false) io.tiles(0).grant_ack.ready := Bool(true) @@ -314,8 +314,8 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val send_grant_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } val do_free_arr = Vec(NGLOBAL_XACTS){ Bool() } - val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } - val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } + val release_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } + val probe_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } val sent_grant_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } val p_data_client_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } @@ -332,8 +332,8 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co t.grant_ack := do_free_arr(i) t.p_data.bits.client_id := p_data_client_id_arr(i) t.p_data.valid := p_data_valid_arr(i) - t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits - t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits + t.release_cnt_dec := release_cnt_dec_arr(i).toBits + t.probe_cnt_inc := probe_cnt_inc_arr(i).toBits t.tile_incoherent := io.incoherent.toBits t.sent_grant_ack := sent_grant_ack_arr(i) do_free_arr(i) := Bool(false) @@ -341,21 +341,21 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co p_data_client_id_arr(i) := Bits(0, width = conf.ln.idBits) p_data_valid_arr(i) := Bool(false) for( j <- 0 until conf.ln.nTiles) { - p_rep_cnt_dec_arr(i)(j) := Bool(false) - p_req_cnt_inc_arr(i)(j) := Bool(false) + release_cnt_dec_arr(i)(j) := Bool(false) + probe_cnt_inc_arr(i)(j) := Bool(false) } } - val p_rep_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY - val x_init_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY + val release_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY + val acquire_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY // Free finished transactions for( j <- 0 until conf.ln.nTiles ) { - val finish = io.tiles(j).grant_ack - when (finish.valid) { - do_free_arr(finish.bits.payload.master_xact_id) := Bool(true) + val ack = io.tiles(j).grant_ack + when (ack.valid) { + do_free_arr(ack.bits.payload.master_xact_id) := Bool(true) } - finish.ready := Bool(true) + ack.ready := Bool(true) } // Reply to initial requestor @@ -404,31 +404,31 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co // Handle probe replies, which may or may not have data for( j <- 0 until conf.ln.nTiles ) { - val p_rep = io.tiles(j).release - val p_rep_data = io.tiles(j).release_data - val idx = p_rep.bits.payload.master_xact_id - val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) - val do_pop = foldR(pop_p_reps)(_ || _) - p_rep.ready := Bool(true) - p_rep_data_dep_list(j).io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits.payload) - p_rep_data_dep_list(j).io.enq.bits.master_xact_id := p_rep.bits.payload.master_xact_id - p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) - when (p_rep.valid && co.messageHasData(p_rep.bits.payload)) { + val release = io.tiles(j).release + val release_data = io.tiles(j).release_data + val idx = release.bits.payload.master_xact_id + val pop_releases = trackerList.map(_.io.pop_release(j).toBool) + val do_pop = foldR(pop_releases)(_ || _) + release.ready := Bool(true) + release_data_dep_list(j).io.enq.valid := release.valid && co.messageHasData(release.bits.payload) + release_data_dep_list(j).io.enq.bits.master_xact_id := release.bits.payload.master_xact_id + release_data.ready := foldR(trackerList.map(_.io.pop_release_data(j)))(_ || _) + when (release.valid && co.messageHasData(release.bits.payload)) { p_data_valid_arr(idx) := Bool(true) p_data_client_id_arr(idx) := UFix(j) } - p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) + release_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_release_dep(j).toBool))(_||_) } for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_client_id).release_data.valid - trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_client_id).release_data.bits.payload + trackerList(i).io.release_data.valid := io.tiles(trackerList(i).io.release_client_id).release_data.valid + trackerList(i).io.release_data.bits := io.tiles(trackerList(i).io.release_client_id).release_data.bits.payload - trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_client_id, p_rep_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) - trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_client_id, p_rep_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) + trackerList(i).io.release_data_dep.valid := MuxLookup(trackerList(i).io.release_client_id, release_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> release_data_dep_list(j).io.deq.valid)) + trackerList(i).io.release_data_dep.bits := MuxLookup(trackerList(i).io.release_client_id, release_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> release_data_dep_list(j).io.deq.bits)) for( j <- 0 until conf.ln.nTiles) { - val p_rep = io.tiles(j).release - p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.payload.master_xact_id === UFix(i)) + val release = io.tiles(j).release + release_cnt_dec_arr(i)(j) := release.valid && (release.bits.payload.master_xact_id === UFix(i)) } } @@ -437,33 +437,33 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val abort_state_arr = Vec(conf.ln.nTiles) { Reg(resetVal = s_idle) } val want_to_abort_arr = Vec(conf.ln.nTiles) { Bool() } for( j <- 0 until conf.ln.nTiles ) { - val x_init = io.tiles(j).acquire - val x_init_data = io.tiles(j).acquire_data + val acquire = io.tiles(j).acquire + val acquire_data = io.tiles(j).acquire_data val x_abort = io.tiles(j).abort val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val conflicts = Vec(NGLOBAL_XACTS) { Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.payload.addr) + conflicts(i) := t.busy && acquire.valid && co.isCoherenceConflict(t.addr, acquire.bits.payload.addr) } - x_abort.bits.payload.client_xact_id := x_init.bits.payload.client_xact_id + x_abort.bits.payload.client_xact_id := acquire.bits.payload.client_xact_id x_abort.bits.header.dst := UFix(0) // DNC x_abort.bits.header.src := UFix(0) // DNC - want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits.payload))) + want_to_abort_arr(j) := acquire.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!acquire_data_dep_list(j).io.enq.ready && co.messageHasData(acquire.bits.payload))) x_abort.valid := Bool(false) switch(abort_state_arr(j)) { is(s_idle) { when(want_to_abort_arr(j)) { - when(co.messageHasData(x_init.bits.payload)) { + when(co.messageHasData(acquire.bits.payload)) { abort_state_arr(j) := s_abort_drain } . otherwise { abort_state_arr(j) := s_abort_send } } } - is(s_abort_drain) { // raises x_init_data.ready below - when(x_init_data.valid) { + is(s_abort_drain) { // raises acquire_data.ready below + when(acquire_data.valid) { abort_cnt := abort_cnt + UFix(1) when(abort_cnt === ~UFix(0, width = log2Up(REFILL_CYCLES))) { abort_state_arr(j) := s_abort_send @@ -472,7 +472,7 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co } is(s_abort_send) { // nothing is dequeued for now x_abort.valid := Bool(true) - when(x_abort.ready) { // raises x_init.ready below + when(x_abort.ready) { // raises acquire.ready below abort_state_arr(j) := s_idle } } @@ -490,41 +490,41 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co trackerList(i).io.alloc_req.bits := init_arb.io.out.bits trackerList(i).io.alloc_req.valid := init_arb.io.out.valid - trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_client_id).acquire_data.bits.payload - trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_client_id).acquire_data.valid - trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_client_id, x_init_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) - trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_client_id, x_init_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) + trackerList(i).io.acquire_data.bits := io.tiles(trackerList(i).io.init_client_id).acquire_data.bits.payload + trackerList(i).io.acquire_data.valid := io.tiles(trackerList(i).io.init_client_id).acquire_data.valid + trackerList(i).io.acquire_data_dep.bits := MuxLookup(trackerList(i).io.init_client_id, acquire_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> acquire_data_dep_list(j).io.deq.bits)) + trackerList(i).io.acquire_data_dep.valid := MuxLookup(trackerList(i).io.init_client_id, acquire_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> acquire_data_dep_list(j).io.deq.valid)) } for( j <- 0 until conf.ln.nTiles ) { - val x_init = io.tiles(j).acquire - val x_init_data = io.tiles(j).acquire_data - val x_init_data_dep = x_init_data_dep_list(j).io.deq + val acquire = io.tiles(j).acquire + val acquire_data = io.tiles(j).acquire_data + val acquire_data_dep = acquire_data_dep_list(j).io.deq val x_abort = io.tiles(j).abort - init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid - init_arb.io.in(j).bits.acquire := x_init.bits.payload + init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && acquire.valid + init_arb.io.in(j).bits.acquire := acquire.bits.payload init_arb.io.in(j).bits.client_id := UFix(j) - val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) - val do_pop = foldR(pop_x_inits)(_||_) - x_init_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(x_init.bits.payload) && (abort_state_arr(j) === s_idle) - x_init_data_dep_list(j).io.enq.bits.master_xact_id := OHToUFix(pop_x_inits) - x_init.ready := (x_abort.valid && x_abort.ready) || do_pop - x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) - x_init_data_dep.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) + val pop_acquires = trackerList.map(_.io.pop_acquire(j).toBool) + val do_pop = foldR(pop_acquires)(_||_) + acquire_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(acquire.bits.payload) && (abort_state_arr(j) === s_idle) + acquire_data_dep_list(j).io.enq.bits.master_xact_id := OHToUFix(pop_acquires) + acquire.ready := (x_abort.valid && x_abort.ready) || do_pop + acquire_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_acquire_data(j).toBool))(_||_) + acquire_data_dep.ready := foldR(trackerList.map(_.io.pop_acquire_dep(j).toBool))(_||_) } alloc_arb.io.out.ready := init_arb.io.out.valid // Handle probe request generation // Must arbitrate for each request port - val p_req_arb_arr = List.fill(conf.ln.nTiles)((new Arbiter(NGLOBAL_XACTS)) { new Probe() }) + val probe_arb_arr = List.fill(conf.ln.nTiles)((new Arbiter(NGLOBAL_XACTS)) { new Probe() }) for( j <- 0 until conf.ln.nTiles ) { for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io - p_req_arb_arr(j).io.in(i).bits := t.probe.bits - p_req_arb_arr(j).io.in(i).valid := t.probe.valid && t.push_p_req(j) - p_req_cnt_inc_arr(i)(j) := p_req_arb_arr(j).io.in(i).ready + probe_arb_arr(j).io.in(i).bits := t.probe.bits + probe_arb_arr(j).io.in(i).valid := t.probe.valid && t.push_probe(j) + probe_cnt_inc_arr(i)(j) := probe_arb_arr(j).io.in(i).ready } - FIFOedLogicalNetworkIOWrapper(p_req_arb_arr(j).io.out) <> io.tiles(j).probe + FIFOedLogicalNetworkIOWrapper(probe_arb_arr(j).io.out) <> io.tiles(j).probe } } @@ -542,8 +542,8 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren implicit val lnConf = conf.ln val co = conf.co val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) - val p_rep_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth must >= NPRIMARY - val x_init_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth should >= NPRIMARY + val release_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth must >= NPRIMARY + val acquire_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth should >= NPRIMARY for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i) @@ -556,50 +556,50 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren // Handle transaction initiation requests // Only one allocation per cycle // Init requests may or may not have data - val x_init = io.network.acquire - val x_init_data = io.network.acquire_data + val acquire = io.network.acquire + val acquire_data = io.network.acquire_data val x_abort = io.network.abort - val x_dep_deq = x_init_data_dep_q.io.deq + val x_dep_deq = acquire_data_dep_q.io.deq val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } val abort_state = Reg(resetVal = s_idle) val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val any_conflict = trackerList.map(_.io.has_conflict).reduce(_||_) val all_busy = trackerList.map(_.io.busy).reduce(_&&_) - val want_to_abort = x_init.valid && (any_conflict || all_busy || (!x_init_data_dep_q.io.enq.ready && co.messageHasData(x_init.bits.payload))) + val want_to_abort = acquire.valid && (any_conflict || all_busy || (!acquire_data_dep_q.io.enq.ready && co.messageHasData(acquire.bits.payload))) val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } for( i <- 0 until NGLOBAL_XACTS ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy - trackerList(i).io.x_init.bits := x_init.bits - trackerList(i).io.x_init.valid := (abort_state === s_idle) && !want_to_abort && x_init.valid && alloc_arb.io.in(i).ready + trackerList(i).io.acquire.bits := acquire.bits + trackerList(i).io.acquire.valid := (abort_state === s_idle) && !want_to_abort && acquire.valid && alloc_arb.io.in(i).ready - trackerList(i).io.x_init_data.bits := x_init_data.bits - trackerList(i).io.x_init_data.valid := x_init_data.valid - trackerList(i).io.x_init_data_dep.bits := x_dep_deq.bits - trackerList(i).io.x_init_data_dep.valid := x_dep_deq.valid + trackerList(i).io.acquire_data.bits := acquire_data.bits + trackerList(i).io.acquire_data.valid := acquire_data.valid + trackerList(i).io.acquire_data_dep.bits := x_dep_deq.bits + trackerList(i).io.acquire_data_dep.valid := x_dep_deq.valid } - val pop_x_init = trackerList.map(_.io.x_init.ready).reduce(_||_) - x_init.ready := (x_abort.valid && x_abort.ready) || pop_x_init - x_init_data.ready := (abort_state === s_abort_drain) || trackerList.map(_.io.x_init_data.ready).reduce(_||_) - x_init_data_dep_q.io.enq.valid := pop_x_init && co.messageHasData(x_init.bits.payload) && (abort_state === s_idle) - x_init_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(trackerList.map(_.io.x_init.ready)) - x_dep_deq.ready := trackerList.map(_.io.x_init_data_dep.ready).reduce(_||_) + val pop_acquire = trackerList.map(_.io.acquire.ready).reduce(_||_) + acquire.ready := (x_abort.valid && x_abort.ready) || pop_acquire + acquire_data.ready := (abort_state === s_abort_drain) || trackerList.map(_.io.acquire_data.ready).reduce(_||_) + acquire_data_dep_q.io.enq.valid := pop_acquire && co.messageHasData(acquire.bits.payload) && (abort_state === s_idle) + acquire_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(trackerList.map(_.io.acquire.ready)) + x_dep_deq.ready := trackerList.map(_.io.acquire_data_dep.ready).reduce(_||_) - alloc_arb.io.out.ready := x_init.valid + alloc_arb.io.out.ready := acquire.valid // Nack conflicting transaction init attempts - x_abort.bits.header.dst := x_init.bits.header.src + x_abort.bits.header.dst := acquire.bits.header.src x_abort.bits.header.src := UFix(0) //DNC - x_abort.bits.payload.client_xact_id := x_init.bits.payload.client_xact_id + x_abort.bits.payload.client_xact_id := acquire.bits.payload.client_xact_id x_abort.valid := Bool(false) switch(abort_state) { is(s_idle) { when(want_to_abort) { - abort_state := Mux( co.messageHasData(x_init.bits.payload), s_abort_drain, s_abort_send) + abort_state := Mux( co.messageHasData(acquire.bits.payload), s_abort_drain, s_abort_send) } } - is(s_abort_drain) { // raises x_init_data.ready below - when(x_init_data.valid) { + is(s_abort_drain) { // raises acquire_data.ready below + when(acquire_data.valid) { abort_cnt := abort_cnt + UFix(1) when(abort_cnt === ~UFix(0, width = log2Up(REFILL_CYCLES))) { abort_state := s_abort_send @@ -608,38 +608,38 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren } is(s_abort_send) { // nothing is dequeued for now x_abort.valid := Bool(true) - when(x_abort.ready) { // raises x_init.ready + when(x_abort.ready) { // raises acquire.ready abort_state := s_idle } } } // Handle probe request generation - val p_req_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new Probe }} + val probe_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new Probe }} for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io - p_req_arb.io.in(i).bits := t.p_req.bits - p_req_arb.io.in(i).valid := t.p_req.valid - t.p_req.ready := p_req_arb.io.in(i).ready + probe_arb.io.in(i).bits := t.probe.bits + probe_arb.io.in(i).valid := t.probe.valid + t.probe.ready := probe_arb.io.in(i).ready } - io.network.probe <> p_req_arb.io.out + io.network.probe <> probe_arb.io.out // Handle probe replies, which may or may not have data - val p_rep = io.network.release - val p_rep_data = io.network.release_data - val idx = p_rep.bits.payload.master_xact_id - p_rep.ready := trackerList.map(_.io.p_rep.ready).reduce(_||_) - p_rep_data.ready := trackerList.map(_.io.p_rep_data.ready).reduce(_||_) - p_rep_data_dep_q.io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits.payload) - p_rep_data_dep_q.io.enq.bits.master_xact_id := p_rep.bits.payload.master_xact_id - p_rep_data_dep_q.io.deq.ready := trackerList.map(_.io.p_rep_data_dep.ready).reduce(_||_) + val release = io.network.release + val release_data = io.network.release_data + val idx = release.bits.payload.master_xact_id + release.ready := trackerList.map(_.io.release.ready).reduce(_||_) + release_data.ready := trackerList.map(_.io.release_data.ready).reduce(_||_) + release_data_dep_q.io.enq.valid := release.valid && co.messageHasData(release.bits.payload) + release_data_dep_q.io.enq.bits.master_xact_id := release.bits.payload.master_xact_id + release_data_dep_q.io.deq.ready := trackerList.map(_.io.release_data_dep.ready).reduce(_||_) for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.p_rep_data.valid := p_rep_data.valid - trackerList(i).io.p_rep_data.bits := p_rep_data.bits - trackerList(i).io.p_rep_data_dep.valid := p_rep_data_dep_q.io.deq.valid - trackerList(i).io.p_rep_data_dep.bits := p_rep_data_dep_q.io.deq.bits - trackerList(i).io.p_rep.valid := p_rep.valid && (idx === UFix(i)) - trackerList(i).io.p_rep.bits := p_rep.bits + trackerList(i).io.release_data.valid := release_data.valid + trackerList(i).io.release_data.bits := release_data.bits + trackerList(i).io.release_data_dep.valid := release_data_dep_q.io.deq.valid + trackerList(i).io.release_data_dep.bits := release_data_dep_q.io.deq.bits + trackerList(i).io.release.valid := release.valid && (idx === UFix(i)) + trackerList(i).io.release.bits := release.bits } // Reply to initial requestor @@ -664,11 +664,11 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren } // Free finished transactions - val finish = io.network.grant_ack + val ack = io.network.grant_ack for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.free := finish.valid && (finish.bits.payload.master_xact_id === UFix(i)) + trackerList(i).io.free := ack.valid && (ack.bits.payload.master_xact_id === UFix(i)) } - finish.ready := Bool(true) + ack.ready := Bool(true) // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests @@ -688,20 +688,20 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val co = conf.co implicit val ln = conf.ln val io = new Bundle { - val x_init = (new FIFOIO){(new LogicalNetworkIO) { new Acquire }}.flip - val x_init_data = (new FIFOIO){(new LogicalNetworkIO) { new AcquireData }}.flip - val p_rep = (new FIFOIO){(new LogicalNetworkIO) { new Release }}.flip - val p_rep_data = (new FIFOIO){(new LogicalNetworkIO) { new ReleaseData }}.flip + val acquire = (new FIFOIO){(new LogicalNetworkIO) { new Acquire }}.flip + val acquire_data = (new FIFOIO){(new LogicalNetworkIO) { new AcquireData }}.flip + val release = (new FIFOIO){(new LogicalNetworkIO) { new Release }}.flip + val release_data = (new FIFOIO){(new LogicalNetworkIO) { new ReleaseData }}.flip val free = Bool(INPUT) val tile_incoherent = Bits(INPUT, conf.ln.nTiles) - val p_rep_data_dep = (new FIFOIO) { new TrackerDependency }.flip - val x_init_data_dep = (new FIFOIO) { new TrackerDependency }.flip + val release_data_dep = (new FIFOIO) { new TrackerDependency }.flip + val acquire_data_dep = (new FIFOIO) { new TrackerDependency }.flip val mem_resp = (new FIFOIO) { new MemResp }.flip val mem_req_cmd = (new FIFOIO) { new MemReqCmd } val mem_req_data = (new FIFOIO) { new MemData } val mem_req_lock = Bool(OUTPUT) - val p_req = (new FIFOIO) {(new LogicalNetworkIO) { new Probe }} + val probe = (new FIFOIO) {(new LogicalNetworkIO) { new Probe }} val grant = (new FIFOIO) {(new LogicalNetworkIO) { new Grant }} val busy = Bool(OUTPUT) val has_conflict = Bool(OUTPUT) @@ -713,32 +713,32 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) //TODO: Will need id reg for merged release xacts val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) - val p_rep_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) - val p_req_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) + val release_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) val x_needs_read = Reg(resetVal = Bool(false)) - val x_init_data_needs_write = Reg(resetVal = Bool(false)) - val p_rep_data_needs_write = Reg(resetVal = Bool(false)) + val acquire_data_needs_write = Reg(resetVal = Bool(false)) + val release_data_needs_write = Reg(resetVal = Bool(false)) val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) - val p_req_initial_flags = Bits(width = conf.ln.nTiles) - p_req_initial_flags := Bits(0) + val probe_initial_flags = Bits(width = conf.ln.nTiles) + probe_initial_flags := Bits(0) if (conf.ln.nTiles > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence // TODO: this is hackish; figure out how to do it more systematically val probe_self = co match { - case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.x_init.bits.payload) + case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.acquire.bits.payload) case _ => Bool(false) } - val myflag = Mux(probe_self, Bits(0), UFixToOH(io.x_init.bits.header.src(log2Up(conf.ln.nTiles)-1,0))) - p_req_initial_flags := ~(io.tile_incoherent | myflag) + val myflag = Mux(probe_self, Bits(0), UFixToOH(io.acquire.bits.header.src(log2Up(conf.ln.nTiles)-1,0))) + probe_initial_flags := ~(io.tile_incoherent | myflag) } val all_grants_require_acks = Bool(true) io.busy := state != s_idle - io.has_conflict := co.isCoherenceConflict(xact.addr, io.x_init.bits.payload.addr) && (state != s_idle) + io.has_conflict := co.isCoherenceConflict(xact.addr, io.acquire.bits.payload.addr) && (state != s_idle) io.mem_req_cmd.valid := Bool(false) io.mem_req_cmd.bits.rw := Bool(false) io.mem_req_cmd.bits.addr := xact.addr @@ -746,12 +746,20 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com io.mem_req_data.valid := Bool(false) io.mem_req_data.bits.data := UFix(0) io.mem_req_lock := Bool(false) +<<<<<<< HEAD io.p_req.valid := Bool(false) io.p_req.bits.payload.p_type := co.getProbeType(xact.a_type, UFix(0)) io.p_req.bits.payload.master_xact_id := UFix(id) io.p_req.bits.payload.addr := xact.addr io.p_req.bits.header.dst := UFix(0) io.p_req.bits.header.src := UFix(0) // DNC +======= + io.probe.valid := Bool(false) + io.probe.bits.payload.p_type := co.getProbeType(xact.a_type, UFix(0)) + io.probe.bits.payload.master_xact_id := UFix(id) + io.probe.bits.payload.addr := xact.addr + io.probe.bits.header.dst := UFix(0) +>>>>>>> changed val names in hub to match new tilelink names io.grant.bits.payload.data := io.mem_resp.bits.data io.grant.bits.payload.g_type := co.getGrantType(xact.a_type, init_sharer_cnt_) io.grant.bits.payload.client_xact_id := xact.client_xact_id @@ -760,70 +768,70 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com io.grant.bits.header.dst := init_client_id_ io.grant.bits.header.src := UFix(0) // DNC io.grant.valid := (io.mem_resp.valid && (UFix(id) === io.mem_resp.bits.tag)) - io.x_init.ready := Bool(false) - io.x_init_data.ready := Bool(false) - io.x_init_data_dep.ready := Bool(false) - io.p_rep.ready := Bool(false) - io.p_rep_data.ready := Bool(false) - io.p_rep_data_dep.ready := Bool(false) + io.acquire.ready := Bool(false) + io.acquire_data.ready := Bool(false) + io.acquire_data_dep.ready := Bool(false) + io.release.ready := Bool(false) + io.release_data.ready := Bool(false) + io.release_data_dep.ready := Bool(false) io.mem_resp.ready := io.grant.ready switch (state) { is(s_idle) { - when( io.x_init.valid ) { - xact := io.x_init.bits.payload - init_client_id_ := io.x_init.bits.header.src + when( io.acquire.valid ) { + xact := io.acquire.bits.payload + init_client_id_ := io.acquire.bits.header.src init_sharer_cnt_ := UFix(conf.ln.nTiles) // TODO: Broadcast only - x_init_data_needs_write := co.messageHasData(io.x_init.bits.payload) - x_needs_read := co.needsMemRead(io.x_init.bits.payload.a_type, UFix(0)) - p_req_flags := p_req_initial_flags + acquire_data_needs_write := co.messageHasData(io.acquire.bits.payload) + x_needs_read := co.needsMemRead(io.acquire.bits.payload.a_type, UFix(0)) + probe_flags := probe_initial_flags mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) - io.x_init.ready := Bool(true) + io.acquire.ready := Bool(true) if(conf.ln.nTiles > 1) { - p_rep_count := PopCount(p_req_initial_flags) - state := Mux(p_req_initial_flags.orR, s_probe, s_mem) + release_count := PopCount(probe_initial_flags) + state := Mux(probe_initial_flags.orR, s_probe, s_mem) } else state := s_mem } } is(s_probe) { - val curr_p_id = PriorityEncoder(p_req_flags) - when(p_req_flags.orR) { - io.p_req.valid := Bool(true) - io.p_req.bits.header.dst := curr_p_id + val curr_p_id = PriorityEncoder(probe_flags) + when(probe_flags.orR) { + io.probe.valid := Bool(true) + io.probe.bits.header.dst := curr_p_id } - when(io.p_req.ready) { - p_req_flags := p_req_flags & ~(UFixToOH(curr_p_id)) + when(io.probe.ready) { + probe_flags := probe_flags & ~(UFixToOH(curr_p_id)) } - when(io.p_rep.valid) { - io.p_rep.ready := Bool(true) - if(conf.ln.nTiles > 1) p_rep_count := p_rep_count - UFix(1) - when(p_rep_count === UFix(1)) { + when(io.release.valid) { + io.release.ready := Bool(true) + if(conf.ln.nTiles > 1) release_count := release_count - UFix(1) + when(release_count === UFix(1)) { state := s_mem } - p_rep_data_needs_write := co.messageHasData(io.p_rep.bits.payload) + release_data_needs_write := co.messageHasData(io.release.bits.payload) } } is(s_mem) { - when (p_rep_data_needs_write) { + when (release_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, - io.p_rep_data, - p_rep_data_needs_write, + io.release_data, + release_data_needs_write, p_w_mem_cmd_sent, - io.p_rep_data_dep.ready, - io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.master_xact_id === UFix(id))) - } . elsewhen(x_init_data_needs_write) { + io.release_data_dep.ready, + io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(id))) + } . elsewhen(acquire_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, - io.x_init_data, - x_init_data_needs_write, + io.acquire_data, + acquire_data_needs_write, x_w_mem_cmd_sent, - io.x_init_data_dep.ready, - io.x_init_data_dep.valid && (io.x_init_data_dep.bits.master_xact_id === UFix(id))) + io.acquire_data_dep.ready, + io.acquire_data_dep.valid && (io.acquire_data_dep.bits.master_xact_id === UFix(id))) } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { From 67fc09f62ec7c5487d33fd851ef4c96477add9d3 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 25 Mar 2013 16:20:12 -0700 Subject: [PATCH 170/374] Fixes after merge, and always self probe. --- uncore/src/coherence.scala | 10 ++++---- uncore/src/uncore.scala | 52 ++++++++++++++++++-------------------- 2 files changed, 29 insertions(+), 33 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index b4c4067f..1fa09fe6 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -274,7 +274,7 @@ class MICoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) - def needsSelfProbe(acq: Acquire) = acq.a_type === acquireReadUncached + def needsSelfProbe(acq: Acquire) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -431,7 +431,7 @@ class MEICoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) - def needsSelfProbe(acq: Acquire) = acq.a_type === acquireReadUncached + def needsSelfProbe(acq: Acquire) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -592,7 +592,7 @@ class MSICoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) - def needsSelfProbe(acq: Acquire) = acq.a_type === acquireReadUncached + def needsSelfProbe(acq: Acquire) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -758,7 +758,7 @@ class MESICoherence extends CoherencePolicyWithUncached { def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) - def needsSelfProbe(acq: Acquire) = acq.a_type === acquireReadUncached + def needsSelfProbe(acq: Acquire) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -940,7 +940,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) - def needsSelfProbe(acq: Acquire) = acq.a_type === acquireReadUncached + def needsSelfProbe(acq: Acquire) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 14467f7b..ea3c20c4 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -37,33 +37,31 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) - // Handle transaction initiation requests - // Only one allocation per cycle - // Init requests may or may not have data + // Handle acquire transaction initiation val acquire = io.client.acquire val acquire_data = io.client.acquire_data val acq_dep_deq = acquire_data_dep_q.io.deq val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - val all_busy = trackerList.map(_.io.busy).reduce(_&&_) + val block_acquires = any_acquire_conflict || (!acquire_data_dep_q.io.enq.ready && co.messageHasData(acquire.bits.payload)) val alloc_arb = (new Arbiter(trackerList.size)) { Bool() } for( i <- 0 until trackerList.size ) { - alloc_arb.io.in(i).valid := !trackerList(i).io.busy val t = trackerList(i).io.client + alloc_arb.io.in(i).valid := t.acquire.ready t.acquire.bits := acquire.bits - t.acquire.valid := acquire.valid && alloc_arb.io.in(i).ready + t.acquire.valid := alloc_arb.io.in(i).ready t.acquire_data.bits := acquire_data.bits t.acquire_data.valid := acquire_data.valid trackerList(i).io.acquire_data_dep.bits := acq_dep_deq.bits trackerList(i).io.acquire_data_dep.valid := acq_dep_deq.valid } - acquire.ready := trackerList.map(_.io.client.acquire.ready).reduce(_||_) + acquire.ready := trackerList.map(_.io.client.acquire.ready).reduce(_||_) && !block_acquires acquire_data.ready := trackerList.map(_.io.client.acquire_data.ready).reduce(_||_) - acquire_data_dep_q.io.enq.valid := acquire.ready && co.messageHasData(acquire.bits.payload) - acquire_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(trackerList.map(_.io.client.acquire.ready)) + acquire_data_dep_q.io.enq.valid := acquire.valid && acquire.ready && co.messageHasData(acquire.bits.payload) + acquire_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(alloc_arb.io.in.map(_.ready)) acq_dep_deq.ready := trackerList.map(_.io.acquire_data_dep.ready).reduce(_||_) - alloc_arb.io.out.ready := acquire.valid + alloc_arb.io.out.ready := acquire.valid && !block_acquires // Handle probe request generation val probe_arb = (new Arbiter(trackerList.size)){(new LogicalNetworkIO){ new Probe }} @@ -75,22 +73,23 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends val release_data = io.client.release_data val voluntary = co.isVoluntary(release.bits.payload) val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) + val block_releases = (!release_data_dep_q.io.enq.ready && co.messageHasData(release.bits.payload)) val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)){Bool()}.lastIndexWhere{b: Bool => b} val idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UFix(0)), release.bits.payload.master_xact_id) - release.ready := trackerList.map(_.io.client.release.ready).reduce(_||_) - release_data.ready := trackerList.map(_.io.client.release_data.ready).reduce(_||_) - release_data_dep_q.io.enq.valid := release.valid && co.messageHasData(release.bits.payload) - release_data_dep_q.io.enq.bits.master_xact_id := idx - release_data_dep_q.io.deq.ready := trackerList.map(_.io.release_data_dep.ready).reduce(_||_) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.client - t.release.valid := release.valid && (idx === UFix(i)) t.release.bits := release.bits - t.release_data.valid := release_data.valid + t.release.valid := release.valid && (idx === UFix(i)) && !block_releases t.release_data.bits := release_data.bits - trackerList(i).io.release_data_dep.valid := release_data_dep_q.io.deq.valid + t.release_data.valid := release_data.valid trackerList(i).io.release_data_dep.bits := release_data_dep_q.io.deq.bits + trackerList(i).io.release_data_dep.valid := release_data_dep_q.io.deq.valid } + release.ready := Vec(trackerList.map(_.io.client.release.ready)){Bool()}(idx) && !block_releases + release_data.ready := trackerList.map(_.io.client.release_data.ready).reduce(_||_) + release_data_dep_q.io.enq.valid := release.valid && release.ready && co.messageHasData(release.bits.payload) + release_data_dep_q.io.enq.bits.master_xact_id := idx + release_data_dep_q.io.deq.ready := trackerList.map(_.io.release_data_dep.ready).reduce(_||_) // Reply to initial requestor val grant_arb = (new Arbiter(trackerList.size)){(new LogicalNetworkIO){ new Grant }} @@ -121,7 +120,6 @@ abstract class XactTracker()(implicit conf: UncoreConfiguration) extends Compone val release_data_dep = (new FIFOIO) { new TrackerDependency }.flip val acquire_data_dep = (new FIFOIO) { new TrackerDependency }.flip - val busy = Bool(OUTPUT) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) } @@ -139,7 +137,6 @@ class WritebackTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigu io.acquire_data_dep.ready := Bool(false) io.release_data_dep.ready := Bool(false) - io.busy := Bool(true) io.has_acquire_conflict := Bool(false) io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.bits.payload.addr) && (state != s_idle) @@ -156,7 +153,7 @@ class WritebackTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigu io.client.acquire_data.ready := Bool(false) io.client.probe.valid := Bool(false) io.client.release.ready := Bool(false) - io.client.release_data.ready := Bool(false) + io.client.release_data.ready := Bool(false) // DNC io.client.grant.valid := Bool(false) io.client.grant.bits.payload.g_type := co.getGrantType(xact, UFix(0)) io.client.grant.bits.payload.client_xact_id := xact.client_xact_id @@ -167,13 +164,13 @@ class WritebackTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigu switch (state) { is(s_idle) { + io.client.release.ready := Bool(true) when( io.client.release.valid ) { xact := io.client.release.bits.payload init_client_id_ := io.client.release.bits.header.src release_data_needs_write := co.messageHasData(io.client.release.bits.payload) mem_cnt := UFix(0) mem_cmd_sent := Bool(false) - io.client.release.ready := Bool(true) state := s_mem } } @@ -205,6 +202,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt_) val release_count = if (conf.ln.nClients == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nClients)) + val curr_p_id = PriorityEncoder(probe_flags) val x_needs_read = Reg(resetVal = Bool(false)) val acquire_data_needs_write = Reg(resetVal = Bool(false)) val release_data_needs_write = Reg(resetVal = Bool(false)) @@ -216,12 +214,11 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura probe_initial_flags := Bits(0) if (conf.ln.nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence - val probe_self = co.needsSelfProbe(io.client.acquire.bits.payload) + val probe_self = Bool(true) //co.needsSelfProbe(io.client.acquire.bits.payload) val myflag = Mux(probe_self, Bits(0), UFixToOH(io.client.acquire.bits.header.src(log2Up(conf.ln.nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } - io.busy := state != s_idle io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, io.client.acquire.bits.payload.addr) && (state != s_idle) io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.bits.payload.addr) && (state != s_idle) io.master.acquire.valid := Bool(false) @@ -256,6 +253,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura switch (state) { is(s_idle) { + io.client.acquire.ready := Bool(true) when( io.client.acquire.valid ) { xact := io.client.acquire.bits.payload init_client_id_ := io.client.acquire.bits.header.src @@ -266,7 +264,6 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura mem_cnt := UFix(0) r_w_mem_cmd_sent := Bool(false) a_w_mem_cmd_sent := Bool(false) - io.client.acquire.ready := Bool(true) if(conf.ln.nClients > 1) { release_count := PopCount(probe_initial_flags) state := Mux(probe_initial_flags.orR, s_probe, s_mem) @@ -274,7 +271,6 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura } } is(s_probe) { - val curr_p_id = PriorityEncoder(probe_flags) when(probe_flags.orR) { io.client.probe.valid := Bool(true) io.client.probe.bits.header.dst := curr_p_id @@ -282,13 +278,13 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura when(io.client.probe.ready) { probe_flags := probe_flags & ~(UFixToOH(curr_p_id)) } + io.client.release.ready := Bool(true) when(io.client.release.valid) { - io.client.release.ready := Bool(true) if(conf.ln.nClients > 1) release_count := release_count - UFix(1) when(release_count === UFix(1)) { state := s_mem } - release_data_needs_write := co.messageHasData(io.client.release.bits.payload) + release_data_needs_write := release_data_needs_write || co.messageHasData(io.client.release.bits.payload) } } is(s_mem) { From b6cc08e8ca563617607c07e37c5427f2d2813fef Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 28 Mar 2013 14:09:08 -0700 Subject: [PATCH 171/374] override io in LogicalNetwork --- uncore/src/package.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 492079cc..f353a524 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -49,7 +49,7 @@ class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfi case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nMasters: Int, nClients: Int) abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[CoherenceAgentRole])(implicit conf: LogicalNetworkConfiguration) extends Component { - val io: Vec[TileLinkType] + override val io: Vec[TileLinkType] val physicalNetworks: Seq[PhysicalNetwork] require(endpoints.length == conf.nEndpoints) } From 3479f1c6cd86bc0d555b1cb0c20660a274a098c6 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 3 Apr 2013 22:13:51 -0700 Subject: [PATCH 172/374] add LR/SC support --- uncore/src/coherence.scala | 12 ++++-------- uncore/src/consts.scala | 40 +++++++++++++++++++++----------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index 1fa09fe6..d97cc2da 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -121,8 +121,7 @@ class ThreeStateIncoherence extends IncoherentPolicy { def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, acquireReadDirty, acquireReadClean) + Mux(isWriteIntent(cmd), acquireReadDirty, acquireReadClean) } def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -517,8 +516,7 @@ class MSICoherence extends CoherencePolicyWithUncached { def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, acquireReadExclusive, acquireReadShared) + Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) } def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -679,8 +677,7 @@ class MESICoherence extends CoherencePolicyWithUncached { def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, acquireReadExclusive, acquireReadShared) + Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) } def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -857,8 +854,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), acquireReadShared) + Mux(isWriteIntent(cmd), Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), acquireReadShared) } def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { val (read, write) = cpuCmdToRW(cmd) diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index d348e0ff..cde8d3b9 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -45,27 +45,31 @@ trait MemoryOpConstants { val MT_HU = Bits("b101", 3); val MT_WU = Bits("b110", 3); - val M_X = Bits("b????", 4); - val M_XRD = Bits("b0000", 4); // int load - val M_XWR = Bits("b0001", 4); // int store - val M_PFR = Bits("b0010", 4); // prefetch with intent to read - val M_PFW = Bits("b0011", 4); // prefetch with intent to write - val M_FENCE = Bits("b0101", 4); // memory fence - val M_INV = Bits("b0110", 4); // write back and invalidate line - val M_CLN = Bits("b0111", 4); // write back line - val M_XA_ADD = Bits("b1000", 4); - val M_XA_SWAP = Bits("b1001", 4); - val M_XA_AND = Bits("b1010", 4); - val M_XA_OR = Bits("b1011", 4); - val M_XA_MIN = Bits("b1100", 4); - val M_XA_MAX = Bits("b1101", 4); - val M_XA_MINU = Bits("b1110", 4); - val M_XA_MAXU = Bits("b1111", 4); + val M_SZ = 5 + val M_X = Bits("b?????"); + val M_XRD = Bits("b00000"); // int load + val M_XWR = Bits("b00001"); // int store + val M_PFR = Bits("b00010"); // prefetch with intent to read + val M_PFW = Bits("b00011"); // prefetch with intent to write + val M_FENCE = Bits("b00101"); // memory fence + val M_XLR = Bits("b00110"); + val M_XSC = Bits("b00111"); + val M_XA_ADD = Bits("b01000"); + val M_XA_SWAP = Bits("b01001"); + val M_XA_AND = Bits("b01010"); + val M_XA_OR = Bits("b01011"); + val M_XA_MIN = Bits("b01100"); + val M_XA_MAX = Bits("b01101"); + val M_XA_MINU = Bits("b01110"); + val M_XA_MAXU = Bits("b01111"); + val M_INV = Bits("b10000"); // write back and invalidate line + val M_CLN = Bits("b10001"); // write back line def isAMO(cmd: Bits) = cmd(3) def isPrefetch(cmd: Bits) = cmd === M_PFR || cmd === M_PFW - def isRead(cmd: Bits) = cmd === M_XRD || isAMO(cmd) - def isWrite(cmd: Bits) = cmd === M_XWR || isAMO(cmd) + def isRead(cmd: Bits) = cmd === M_XRD || cmd === M_XLR || isAMO(cmd) + def isWrite(cmd: Bits) = cmd === M_XWR || cmd === M_XSC || isAMO(cmd) + def isWriteIntent(cmd: Bits) = isWrite(cmd) || cmd === M_PFW || cmd === M_XLR } trait MemoryInterfaceConstants extends From 7ff5b5b86f1dfa508b4c67a3d2c1127da7611f61 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sun, 7 Apr 2013 19:23:44 -0700 Subject: [PATCH 173/374] treat load-reserved as a non-dirtying store --- uncore/src/coherence.scala | 66 +++++++++++++------------------------- 1 file changed, 22 insertions(+), 44 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index d97cc2da..20157456 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -7,10 +7,6 @@ trait CoherenceAgentRole trait ClientCoherenceAgent extends CoherenceAgentRole trait MasterCoherenceAgent extends CoherenceAgentRole -object cpuCmdToRW { - def apply(cmd: Bits): (Bool, Bool) = (isRead(cmd) || isPrefetch(cmd), isWrite(cmd)) -} - abstract class CoherencePolicy { def isHit (cmd: Bits, state: UFix): Bool def isValid (state: UFix): Bool @@ -102,8 +98,7 @@ class ThreeStateIncoherence extends IncoherentPolicy { def needsWriteback (state: UFix): Bool = state === tileDirty def newState(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, tileDirty, Mux(read, Mux(state === tileDirty, tileDirty, tileClean), state)) + Mux(isWrite(cmd), tileDirty, Mux(isRead(cmd), Mux(state === tileDirty, tileDirty, tileClean), state)) } def newStateOnHit(cmd: Bits, state: UFix): UFix = newState(cmd, state) def newStateOnCacheControl(cmd: Bits) = tileInvalid //TODO @@ -124,8 +119,7 @@ class ThreeStateIncoherence extends IncoherentPolicy { Mux(isWriteIntent(cmd), acquireReadDirty, acquireReadClean) } def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, acquireReadDirty, outstanding.a_type) + Mux(isWriteIntent(cmd), acquireReadDirty, outstanding.a_type) } def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = releaseVoluntaryInvalidateData @@ -296,9 +290,8 @@ class MEICoherence extends CoherencePolicyWithUncached { def isValid (state: UFix): Bool = state != tileInvalid def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { - val (read, write) = cpuCmdToRW(cmd) - (read && messageIsUncached(outstanding)) || - (write && (outstanding.a_type != acquireReadExclusiveDirty)) + (isRead(cmd) && messageIsUncached(outstanding)) || + (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusiveDirty)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( @@ -311,8 +304,7 @@ class MEICoherence extends CoherencePolicyWithUncached { } def newStateOnHit(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, tileExclusiveDirty, state) + Mux(isWrite(cmd), tileExclusiveDirty, state) } def newStateOnCacheControl(cmd: Bits) = { MuxLookup(cmd, tileInvalid, Array( @@ -352,12 +344,10 @@ class MEICoherence extends CoherencePolicyWithUncached { def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, acquireReadExclusiveDirty, acquireReadExclusiveClean) + Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, acquireReadExclusiveClean) } def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, acquireReadExclusiveDirty, outstanding.a_type) + Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, outstanding.a_type) } def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) @@ -450,8 +440,7 @@ class MSICoherence extends CoherencePolicyWithUncached { val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, (state === tileExclusiveDirty), + Mux(isWriteIntent(cmd), (state === tileExclusiveDirty), (state === tileShared || state === tileExclusiveDirty)) } def isValid (state: UFix): Bool = { @@ -459,9 +448,8 @@ class MSICoherence extends CoherencePolicyWithUncached { } def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { - val (read, write) = cpuCmdToRW(cmd) - (read && messageIsUncached(outstanding)) || - (write && (outstanding.a_type != acquireReadExclusive)) + (isRead(cmd) && messageIsUncached(outstanding)) || + (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( @@ -474,8 +462,7 @@ class MSICoherence extends CoherencePolicyWithUncached { } def newStateOnHit(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, tileExclusiveDirty, state) + Mux(isWrite(cmd), tileExclusiveDirty, state) } def newStateOnCacheControl(cmd: Bits) = { MuxLookup(cmd, tileInvalid, Array( @@ -519,8 +506,7 @@ class MSICoherence extends CoherencePolicyWithUncached { Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) } def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, acquireReadExclusive, outstanding.a_type) + Mux(isWriteIntent(cmd), acquireReadExclusive, outstanding.a_type) } def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) @@ -611,8 +597,7 @@ class MESICoherence extends CoherencePolicyWithUncached { val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, (state === tileExclusiveClean || state === tileExclusiveDirty), + Mux(isWriteIntent(cmd), (state === tileExclusiveClean || state === tileExclusiveDirty), (state === tileShared || state === tileExclusiveClean || state === tileExclusiveDirty)) } def isValid (state: UFix): Bool = { @@ -620,9 +605,8 @@ class MESICoherence extends CoherencePolicyWithUncached { } def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { - val (read, write) = cpuCmdToRW(cmd) - (read && messageIsUncached(outstanding)) || - (write && (outstanding.a_type != acquireReadExclusive)) + (isRead(cmd) && messageIsUncached(outstanding)) || + (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( @@ -635,8 +619,7 @@ class MESICoherence extends CoherencePolicyWithUncached { } def newStateOnHit(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, tileExclusiveDirty, state) + Mux(isWrite(cmd), tileExclusiveDirty, state) } def newStateOnCacheControl(cmd: Bits) = { MuxLookup(cmd, tileInvalid, Array( @@ -680,8 +663,7 @@ class MESICoherence extends CoherencePolicyWithUncached { Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) } def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, acquireReadExclusive, outstanding.a_type) + Mux(isWriteIntent(cmd), acquireReadExclusive, outstanding.a_type) } def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) @@ -775,17 +757,15 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) def isHit (cmd: Bits, state: UFix): Bool = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, uFixListContains(List(tileExclusiveClean, tileExclusiveDirty, tileMigratoryClean, tileMigratoryDirty), state), (state != tileInvalid)) + Mux(isWriteIntent(cmd), uFixListContains(List(tileExclusiveClean, tileExclusiveDirty, tileMigratoryClean, tileMigratoryDirty), state), (state != tileInvalid)) } def isValid (state: UFix): Bool = { state != tileInvalid } def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { - val (read, write) = cpuCmdToRW(cmd) - (read && messageIsUncached(outstanding)) || - (write && (outstanding.a_type != acquireReadExclusive && outstanding.a_type != acquireInvalidateOthers)) + (isRead(cmd) && messageIsUncached(outstanding)) || + (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive && outstanding.a_type != acquireInvalidateOthers)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( @@ -798,8 +778,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def newStateOnHit(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, MuxLookup(state, tileExclusiveDirty, Array( + Mux(isWrite(cmd), MuxLookup(state, tileExclusiveDirty, Array( tileExclusiveClean -> tileExclusiveDirty, tileMigratoryClean -> tileMigratoryDirty)), state) } @@ -857,8 +836,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { Mux(isWriteIntent(cmd), Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), acquireReadShared) } def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), outstanding.a_type) + Mux(isWriteIntent(cmd), Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), outstanding.a_type) } def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) From 74187c2068144b640347fa674014bfb868e3428e Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 9 Apr 2013 14:09:36 -0700 Subject: [PATCH 174/374] Always route voluntary releases to ReleaseTracker to ensure all grants are sent --- uncore/src/uncore.scala | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index ea3c20c4..3e54b9e3 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -68,27 +68,28 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends io.client.probe <> probe_arb.io.out probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.probe } - // Handle probe replies, which may or may not have data + // Handle releases, which might be voluntary and might have data val release = io.client.release val release_data = io.client.release_data val voluntary = co.isVoluntary(release.bits.payload) val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) val block_releases = (!release_data_dep_q.io.enq.ready && co.messageHasData(release.bits.payload)) val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)){Bool()}.lastIndexWhere{b: Bool => b} - val idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UFix(0)), release.bits.payload.master_xact_id) + //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UFix(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response + val release_idx = Mux(voluntary, UFix(0), release.bits.payload.master_xact_id) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.client t.release.bits := release.bits - t.release.valid := release.valid && (idx === UFix(i)) && !block_releases + t.release.valid := release.valid && (release_idx === UFix(i)) && !block_releases t.release_data.bits := release_data.bits t.release_data.valid := release_data.valid trackerList(i).io.release_data_dep.bits := release_data_dep_q.io.deq.bits trackerList(i).io.release_data_dep.valid := release_data_dep_q.io.deq.valid } - release.ready := Vec(trackerList.map(_.io.client.release.ready)){Bool()}(idx) && !block_releases + release.ready := Vec(trackerList.map(_.io.client.release.ready)){Bool()}(release_idx) && !block_releases release_data.ready := trackerList.map(_.io.client.release_data.ready).reduce(_||_) release_data_dep_q.io.enq.valid := release.valid && release.ready && co.messageHasData(release.bits.payload) - release_data_dep_q.io.enq.bits.master_xact_id := idx + release_data_dep_q.io.enq.bits.master_xact_id := release_idx release_data_dep_q.io.deq.ready := trackerList.map(_.io.release_data_dep.ready).reduce(_||_) // Reply to initial requestor From 766d5622b17d432286a11c1a98ee46423ad55912 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 10 Apr 2013 13:46:31 -0700 Subject: [PATCH 175/374] Prevent messages from becoming interleaved in the BasicCrossbar. Remove dependency trackers from the uncore, use msg headers instead. Have one ReleaseHnadler per core for now. --- uncore/src/coherence.scala | 10 ++--- uncore/src/package.scala | 53 +++++++++++++++++++++++- uncore/src/tilelink.scala | 5 ++- uncore/src/uncore.scala | 82 +++++++++++++------------------------- 4 files changed, 86 insertions(+), 64 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index 20157456..ec932ec9 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -265,7 +265,7 @@ class MICoherence extends CoherencePolicyWithUncached { def needsAckReply(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } - def requiresAck(grant: Grant) = Bool(true) + def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck def requiresAck(release: Release) = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) @@ -418,7 +418,7 @@ class MEICoherence extends CoherencePolicyWithUncached { def needsAckReply(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } - def requiresAck(grant: Grant) = Bool(true) + def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck def requiresAck(release: Release) = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) @@ -574,7 +574,7 @@ class MSICoherence extends CoherencePolicyWithUncached { def needsAckReply(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } - def requiresAck(grant: Grant) = Bool(true) + def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck def requiresAck(release: Release) = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) @@ -735,7 +735,7 @@ class MESICoherence extends CoherencePolicyWithUncached { (a_type === acquireWriteUncached) } - def requiresAck(grant: Grant) = Bool(true) + def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck def requiresAck(release: Release) = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) @@ -912,7 +912,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def needsAckReply(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached ||a_type === acquireInvalidateOthers) } - def requiresAck(grant: Grant) = Bool(true) + def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck def requiresAck(release: Release) = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index f353a524..4133a9d3 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -24,7 +24,56 @@ class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkCon abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Component -class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { +class LockingRRArbiter[T <: Data](n: Int, count: Int)(data: => T) extends Component { + val io = new ioArbiter(n)(data) + + val cnt = Reg(resetVal = UFix(0, width = log2Up(count))) + val cnt_next = cnt + UFix(1) + val locked = Reg(resetVal = Bool(false)) + val lock_idx = Reg(resetVal = UFix(n)) + + if(count > 1){ + when(io.out.valid && io.out.ready) { + cnt := cnt_next + when(!locked) { + locked := Bool(true) + lock_idx := Vec(io.in.map{ in => in.ready && in.valid}){Bool()}.indexWhere{i: Bool => i} + } + when(cnt_next === UFix(0)) { + locked := Bool(false) + } + } + } else { + locked := Bool(false) + lock_idx := UFix(n) + cnt := UFix(0) + } + + + val last_grant = Reg(resetVal = Bits(0, log2Up(n))) + val g = ArbiterCtrl((0 until n).map(i => io.in(i).valid && UFix(i) > last_grant) ++ io.in.map(_.valid)) + val grant = (0 until n).map(i => g(i) && UFix(i) > last_grant || g(i+n)) + (0 until n).map(i => io.in(i).ready := Mux(locked, lock_idx === UFix(i), grant(i)) && io.out.ready) + + var choose = Bits(n-1) + for (i <- n-2 to 0 by -1) + choose = Mux(io.in(i).valid, Bits(i), choose) + for (i <- n-1 to 1 by -1) + choose = Mux(io.in(i).valid && UFix(i) > last_grant, Bits(i), choose) + choose = Mux(locked, lock_idx, choose) + when (io.out.valid && io.out.ready) { + last_grant := choose + } + + val dvec = Vec(n) { data } + (0 until n).map(i => dvec(i) := io.in(i).bits ) + + io.out.valid := Mux(locked, io.in(lock_idx).valid, io.in.map(_.valid).foldLeft(Bool(false))( _ || _)) + io.out.bits := dvec(choose) + io.chosen := choose +} + +class BasicCrossbar[T <: Data](count: Int)(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { val io = new Bundle { val in = Vec(conf.nEndpoints){(new FIFOIO){(new BasicCrossbarIO){data}}}.flip val out = Vec(conf.nEndpoints){(new FIFOIO){(new BasicCrossbarIO){data}}} @@ -33,7 +82,7 @@ class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfi val rdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { - val rrarb = (new RRArbiter(conf.nEndpoints)){io.in(0).bits.clone} + val rrarb = (new LockingRRArbiter(conf.nEndpoints, count)){io.in(0).bits.clone} (rrarb.io.in, io.in, rdys).zipped.map{ case (arb, in, rdy) => { arb.valid := in.valid && (in.bits.header.dst === UFix(i)) arb.bits := in.bits diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index c16f8c9b..c7f82da2 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -118,11 +118,12 @@ class GrantAck extends ClientSourcedMessage with HasMasterTransactionId abstract class DirectionalFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) class ClientSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) +class ClientSourcedDataIO[T <: Data]()(data: => T) extends ClientSourcedIO()(data) class MasterSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) {flip()} class UncachedTileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { val acquire = (new ClientSourcedIO){(new LogicalNetworkIO){new Acquire }} - val acquire_data = (new ClientSourcedIO){(new LogicalNetworkIO){new AcquireData }} + val acquire_data = (new ClientSourcedDataIO){(new LogicalNetworkIO){new AcquireData }} val grant = (new MasterSourcedIO) {(new LogicalNetworkIO){new Grant }} val grant_ack = (new ClientSourcedIO){(new LogicalNetworkIO){new GrantAck }} override def clone = { new UncachedTileLinkIO().asInstanceOf[this.type] } @@ -131,7 +132,7 @@ class UncachedTileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bun class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends UncachedTileLinkIO()(conf) { val probe = (new MasterSourcedIO){(new LogicalNetworkIO){new Probe }} val release = (new ClientSourcedIO){(new LogicalNetworkIO){new Release }} - val release_data = (new ClientSourcedIO){(new LogicalNetworkIO){new ReleaseData }} + val release_data = (new ClientSourcedDataIO){(new LogicalNetworkIO){new ReleaseData }} override def clone = { new TileLinkIO().asInstanceOf[this.type] } } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 3e54b9e3..445444db 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -3,18 +3,10 @@ package uncore import Chisel._ import Constants._ -class TrackerProbeData(implicit conf: UncoreConfiguration) extends Bundle { - val client_id = Bits(width = conf.ln.idBits) -} - -class TrackerAllocReq(implicit conf: UncoreConfiguration) extends Bundle { - val acquire = new Acquire() - val client_id = Bits(width = conf.ln.idBits) - override def clone = { new TrackerAllocReq().asInstanceOf[this.type] } -} - -class TrackerDependency extends Bundle { - val master_xact_id = Bits(width = MASTER_XACT_ID_MAX_BITS) +class TrackerDependency(implicit conf: UncoreConfiguration) extends Bundle { + val tracker_id = Bits(width = MASTER_XACT_ID_MAX_BITS) + val data_src_id = Bits(width = conf.ln.idBits) + override def clone = { new TrackerDependency().asInstanceOf[this.type] } } case class UncoreConfiguration(co: CoherencePolicyWithUncached, ln: LogicalNetworkConfiguration) @@ -31,18 +23,16 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends { implicit val lnConf = conf.ln val co = conf.co - val trackerList = new WritebackTracker(0, bankId) +: (1 to NGLOBAL_XACTS).map(new AcquireTracker(_, bankId)) - val release_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth must >= NPRIMARY - val acquire_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth should >= NPRIMARY + val trackerList = (0 until conf.ln.nClients).map(new VoluntaryReleaseTracker(_, bankId)) ++ + (conf.ln.nClients until conf.ln.nClients + NGLOBAL_XACTS).map(new AcquireTracker(_, bankId)) trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) // Handle acquire transaction initiation val acquire = io.client.acquire val acquire_data = io.client.acquire_data - val acq_dep_deq = acquire_data_dep_q.io.deq val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - val block_acquires = any_acquire_conflict || (!acquire_data_dep_q.io.enq.ready && co.messageHasData(acquire.bits.payload)) + val block_acquires = any_acquire_conflict val alloc_arb = (new Arbiter(trackerList.size)) { Bool() } for( i <- 0 until trackerList.size ) { @@ -53,14 +43,9 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends t.acquire_data.bits := acquire_data.bits t.acquire_data.valid := acquire_data.valid - trackerList(i).io.acquire_data_dep.bits := acq_dep_deq.bits - trackerList(i).io.acquire_data_dep.valid := acq_dep_deq.valid } acquire.ready := trackerList.map(_.io.client.acquire.ready).reduce(_||_) && !block_acquires acquire_data.ready := trackerList.map(_.io.client.acquire_data.ready).reduce(_||_) - acquire_data_dep_q.io.enq.valid := acquire.valid && acquire.ready && co.messageHasData(acquire.bits.payload) - acquire_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(alloc_arb.io.in.map(_.ready)) - acq_dep_deq.ready := trackerList.map(_.io.acquire_data_dep.ready).reduce(_||_) alloc_arb.io.out.ready := acquire.valid && !block_acquires // Handle probe request generation @@ -73,24 +58,19 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends val release_data = io.client.release_data val voluntary = co.isVoluntary(release.bits.payload) val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) - val block_releases = (!release_data_dep_q.io.enq.ready && co.messageHasData(release.bits.payload)) + val block_releases = Bool(false) val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)){Bool()}.lastIndexWhere{b: Bool => b} //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UFix(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response - val release_idx = Mux(voluntary, UFix(0), release.bits.payload.master_xact_id) + val release_idx = Mux(voluntary, release.bits.header.src, release.bits.payload.master_xact_id) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.client t.release.bits := release.bits t.release.valid := release.valid && (release_idx === UFix(i)) && !block_releases t.release_data.bits := release_data.bits t.release_data.valid := release_data.valid - trackerList(i).io.release_data_dep.bits := release_data_dep_q.io.deq.bits - trackerList(i).io.release_data_dep.valid := release_data_dep_q.io.deq.valid } release.ready := Vec(trackerList.map(_.io.client.release.ready)){Bool()}(release_idx) && !block_releases release_data.ready := trackerList.map(_.io.client.release_data.ready).reduce(_||_) - release_data_dep_q.io.enq.valid := release.valid && release.ready && co.messageHasData(release.bits.payload) - release_data_dep_q.io.enq.bits.master_xact_id := release_idx - release_data_dep_q.io.deq.ready := trackerList.map(_.io.release_data_dep.ready).reduce(_||_) // Reply to initial requestor val grant_arb = (new Arbiter(trackerList.size)){(new LogicalNetworkIO){ new Grant }} @@ -116,17 +96,13 @@ abstract class XactTracker()(implicit conf: UncoreConfiguration) extends Compone val io = new Bundle { val client = (new TileLinkIO).flip val master = new UncachedTileLinkIO - val tile_incoherent = Bits(INPUT, conf.ln.nClients) - val release_data_dep = (new FIFOIO) { new TrackerDependency }.flip - val acquire_data_dep = (new FIFOIO) { new TrackerDependency }.flip - val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) } } -class WritebackTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfiguration) extends XactTracker()(conf) { +class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfiguration) extends XactTracker()(conf) { val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(4){ UFix() } val state = Reg(resetVal = s_idle) val xact = Reg{ new Release } @@ -136,8 +112,6 @@ class WritebackTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigu val cmd_to_write = co.getUncachedWriteAcquire(xact.addr, UFix(trackerId)) val cmd_to_read = co.getUncachedReadAcquire(xact.addr, UFix(trackerId)) - io.acquire_data_dep.ready := Bool(false) - io.release_data_dep.ready := Bool(false) io.has_acquire_conflict := Bool(false) io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.bits.payload.addr) && (state != s_idle) @@ -159,6 +133,7 @@ class WritebackTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigu io.client.grant.bits.payload.g_type := co.getGrantType(xact, UFix(0)) io.client.grant.bits.payload.client_xact_id := xact.client_xact_id io.client.grant.bits.payload.master_xact_id := UFix(trackerId) + io.client.grant.bits.payload.data := UFix(0) io.client.grant.bits.header.dst := init_client_id_ io.client.grant.bits.header.src := UFix(bankId) io.client.grant_ack.valid := Bool(false) @@ -181,9 +156,8 @@ class WritebackTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigu io.master.acquire_data, io.client.release_data, release_data_needs_write, - mem_cmd_sent, - io.release_data_dep.ready, - io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(trackerId))) + mem_cmd_sent, + init_client_id_) } . otherwise { state := s_ack } } is(s_ack) { @@ -198,6 +172,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura val state = Reg(resetVal = s_idle) val xact = Reg{ new Acquire } val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) + val release_data_client_id = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) //TODO: Will need id reg for merged release xacts val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt_) @@ -245,10 +220,8 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura io.client.grant.valid := (io.master.grant.valid && (UFix(trackerId) === io.master.grant.bits.payload.client_xact_id)) io.client.acquire.ready := Bool(false) io.client.acquire_data.ready := Bool(false) - io.acquire_data_dep.ready := Bool(false) io.client.release.ready := Bool(false) io.client.release_data.ready := Bool(false) - io.release_data_dep.ready := Bool(false) io.master.grant.ready := io.client.grant.ready io.client.grant_ack.valid := Bool(false) @@ -285,7 +258,10 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura when(release_count === UFix(1)) { state := s_mem } - release_data_needs_write := release_data_needs_write || co.messageHasData(io.client.release.bits.payload) + when( co.messageHasData(io.client.release.bits.payload)) { + release_data_needs_write := Bool(true) + release_data_client_id := io.client.release.bits.header.src + } } } is(s_mem) { @@ -295,16 +271,14 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura io.client.release_data, release_data_needs_write, r_w_mem_cmd_sent, - io.release_data_dep.ready, - io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(trackerId))) + release_data_client_id) } . elsewhen(acquire_data_needs_write) { doOuterReqWrite(io.master.acquire, io.master.acquire_data, io.client.acquire_data, acquire_data_needs_write, a_w_mem_cmd_sent, - io.acquire_data_dep.ready, - io.acquire_data_dep.valid && (io.acquire_data_dep.bits.master_xact_id === UFix(trackerId))) + init_client_id_) } . elsewhen (x_needs_read) { doOuterReqRead(io.master.acquire, x_needs_read) } . otherwise { @@ -330,24 +304,22 @@ abstract trait OuterRequestGenerator { val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) - def doOuterReqWrite[T <: Data](master_acq: FIFOIO[LogicalNetworkIO[Acquire]], master_acq_data: FIFOIO[LogicalNetworkIO[AcquireData]], client_data: FIFOIO[LogicalNetworkIO[T]], trigger: Bool, cmd_sent: Bool, pop_dep: Bool, at_front_of_dep_queue: Bool) { + def doOuterReqWrite[T <: Data](master_acq: FIFOIO[LogicalNetworkIO[Acquire]], master_acq_data: FIFOIO[LogicalNetworkIO[AcquireData]], client_data: FIFOIO[LogicalNetworkIO[T]], trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UFix) { + val do_write = client_data.valid && (client_data.bits.header.src === desired_client_data_src_id) master_acq.bits.payload := cmd_to_write master_acq_data.bits.payload := client_data.bits.payload when(master_acq.ready && master_acq.valid) { cmd_sent := Bool(true) } - when (at_front_of_dep_queue) { - master_acq.valid := !cmd_sent && master_acq_data.ready && client_data.valid + when (do_write) { + master_acq.valid := !cmd_sent && master_acq_data.ready when (master_acq.ready || cmd_sent) { master_acq_data.valid := client_data.valid when(master_acq_data.ready) { client_data.ready:= Bool(true) - when (client_data.valid) { - mem_cnt := mem_cnt_next - when(mem_cnt === UFix(REFILL_CYCLES-1)) { - pop_dep := Bool(true) - trigger := Bool(false) - } + mem_cnt := mem_cnt_next + when(mem_cnt === UFix(REFILL_CYCLES-1)) { + trigger := Bool(false) } } } From 12d394811efccb7e1f83b3935695235b24be447d Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 29 Apr 2013 18:47:37 -0700 Subject: [PATCH 176/374] Allow release data to be written out even before all releases have been collected --- uncore/src/uncore.scala | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 445444db..5dcb192f 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -263,6 +263,14 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura release_data_client_id := io.client.release.bits.header.src } } + when (release_data_needs_write) { + doOuterReqWrite(io.master.acquire, + io.master.acquire_data, + io.client.release_data, + release_data_needs_write, + r_w_mem_cmd_sent, + release_data_client_id) + } } is(s_mem) { when (release_data_needs_write) { From fedc2753e4c54476f34c4ff1b559c1b8baf40b46 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 30 Apr 2013 11:00:23 -0700 Subject: [PATCH 177/374] make sure master_xact_id field is large enough for temporary extra release trackers --- uncore/src/consts.scala | 5 +++-- uncore/src/uncore.scala | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index cde8d3b9..9d343b3f 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -10,8 +10,9 @@ abstract trait CoherenceConfigConstants { } trait UncoreConstants { - val NGLOBAL_XACTS = 8 - val MASTER_XACT_ID_MAX_BITS = log2Up(NGLOBAL_XACTS) + val NGLOBAL_ACQ_XACTS = 8 + val NGLOBAL_REL_XACTS = 4 + val MASTER_XACT_ID_MAX_BITS = log2Up(NGLOBAL_ACQ_XACTS+NGLOBAL_REL_XACTS) val CACHE_DATA_SIZE_IN_BYTES = 1 << 6 } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 5dcb192f..f0571d89 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -23,8 +23,9 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends { implicit val lnConf = conf.ln val co = conf.co - val trackerList = (0 until conf.ln.nClients).map(new VoluntaryReleaseTracker(_, bankId)) ++ - (conf.ln.nClients until conf.ln.nClients + NGLOBAL_XACTS).map(new AcquireTracker(_, bankId)) + require(conf.ln.nClients < NGLOBAL_REL_XACTS) //TODO: handle in config + val trackerList = (0 until NGLOBAL_REL_XACTS).map(new VoluntaryReleaseTracker(_, bankId)) ++ + (NGLOBAL_REL_XACTS until NGLOBAL_REL_XACTS + NGLOBAL_ACQ_XACTS).map(new AcquireTracker(_, bankId)) trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) From 9a258e7fb467e5d3135fc7b372ef73cb9428ede5 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 22 Apr 2013 16:48:55 -0700 Subject: [PATCH 178/374] use new locking round robin arbiter --- uncore/src/package.scala | 51 ++-------------------------------------- 1 file changed, 2 insertions(+), 49 deletions(-) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 4133a9d3..7e1fc7a0 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -6,6 +6,8 @@ import scala.collection.mutable.Stack //TODO: Remove these Networking classes from the package object once Scala bug //SI-3439 is resolved. +implicit def toOption[A](a: A) = Option(a) + case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { @@ -24,55 +26,6 @@ class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkCon abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Component -class LockingRRArbiter[T <: Data](n: Int, count: Int)(data: => T) extends Component { - val io = new ioArbiter(n)(data) - - val cnt = Reg(resetVal = UFix(0, width = log2Up(count))) - val cnt_next = cnt + UFix(1) - val locked = Reg(resetVal = Bool(false)) - val lock_idx = Reg(resetVal = UFix(n)) - - if(count > 1){ - when(io.out.valid && io.out.ready) { - cnt := cnt_next - when(!locked) { - locked := Bool(true) - lock_idx := Vec(io.in.map{ in => in.ready && in.valid}){Bool()}.indexWhere{i: Bool => i} - } - when(cnt_next === UFix(0)) { - locked := Bool(false) - } - } - } else { - locked := Bool(false) - lock_idx := UFix(n) - cnt := UFix(0) - } - - - val last_grant = Reg(resetVal = Bits(0, log2Up(n))) - val g = ArbiterCtrl((0 until n).map(i => io.in(i).valid && UFix(i) > last_grant) ++ io.in.map(_.valid)) - val grant = (0 until n).map(i => g(i) && UFix(i) > last_grant || g(i+n)) - (0 until n).map(i => io.in(i).ready := Mux(locked, lock_idx === UFix(i), grant(i)) && io.out.ready) - - var choose = Bits(n-1) - for (i <- n-2 to 0 by -1) - choose = Mux(io.in(i).valid, Bits(i), choose) - for (i <- n-1 to 1 by -1) - choose = Mux(io.in(i).valid && UFix(i) > last_grant, Bits(i), choose) - choose = Mux(locked, lock_idx, choose) - when (io.out.valid && io.out.ready) { - last_grant := choose - } - - val dvec = Vec(n) { data } - (0 until n).map(i => dvec(i) := io.in(i).bits ) - - io.out.valid := Mux(locked, io.in(lock_idx).valid, io.in.map(_.valid).foldLeft(Bool(false))( _ || _)) - io.out.bits := dvec(choose) - io.chosen := choose -} - class BasicCrossbar[T <: Data](count: Int)(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { val io = new Bundle { val in = Vec(conf.nEndpoints){(new FIFOIO){(new BasicCrossbarIO){data}}}.flip From 9a3b2e7006c370b6c37bee57272a0d1788d50e5b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 1 May 2013 16:48:01 -0700 Subject: [PATCH 179/374] new paired meta/data IO type, and matching arbiter --- uncore/src/package.scala | 62 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 7e1fc7a0..46c39d51 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -8,6 +8,68 @@ import scala.collection.mutable.Stack implicit def toOption[A](a: A) = Option(a) +class PairedDataIO[M <: Data, D <: Data]()(m: => M, d: => D) extends Bundle { + val meta = (new FIFOIO()){ m } + val data = (new FIFOIO()){ d } +} + +class PairedArbiterIO[M <: Data, D <: Data](n: Int)(m: => M, d: => D) extends Bundle { + val in = Vec(n) { new PairedDataIO()(m,d) }.flip + val out = new PairedDataIO()(m,d) + val meta_chosen = Bits(OUTPUT, log2Up(n)) + val data_chosen = Bits(OUTPUT, log2Up(n)) +} + +class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock: Option[M => Bool] = None)(meta: => M, data: => D) { + require(isPow2(count)) + val io = new PairedArbiterIO(n)(meta,data) + val locked = if(count > 1) Reg(resetVal = Bool(false)) else Bool(false) + val lockIdx = if(count > 1) Reg(resetVal = UFix(n-1)) else UFix(n-1) + val grant = List.fill(n)(Bool()) + val meta_chosen = Bits(width = log2Up(n)) + + val chosen_meta_has_data = io.out.meta.valid && needsLock.map(_(io.out.meta.bits)).getOrElse(Bool(true)) + (0 until n).map(i => io.in(i).meta.ready := grant(i) && io.out.meta.ready) + (0 until n).map(i => io.in(i).data.ready := Mux(locked, lockIdx === UFix(i), grant(i) && chosen_meta_has_data) && io.out.data.ready) + io.out.meta.valid := io.in(meta_chosen).meta.valid + io.out.data.valid := Mux(locked, io.in(lockIdx).data.valid, io.in(meta_chosen).data.valid && chosen_meta_has_data) + io.out.meta.bits := io.in(meta_chosen).meta.bits + io.out.data.bits := Mux(locked, io.in(lockIdx).data.bits, io.in(meta_chosen).data.bits) + io.meta_chosen := meta_chosen + io.data_chosen := Mux(locked, lockIdx, meta_chosen) + + if(count > 1){ + val cnt = Reg(resetVal = UFix(0, width = log2Up(count))) + val cnt_next = cnt + UFix(1) + when(io.out.data.fire()){ + cnt := cnt_next + when(cnt_next === UFix(0)) { + locked := Bool(false) + } + } + when(io.out.meta.fire()) { + when(needsLock.map(_(io.out.meta.bits)).getOrElse(Bool(true))) { + when(!locked) { + locked := Bool(true) + lockIdx := Vec(io.in.map{in => in.meta.fire()}){Bool()}.indexWhere{i: Bool => i} + } + } + } + } + val last_grant = Reg(resetVal = Bits(0, log2Up(n))) + val ctrl = ArbiterCtrl((0 until n).map(i => io.in(i).meta.valid && UFix(i) > last_grant) ++ io.in.map(_.meta.valid)) + (0 until n).map(i => grant(i) := ctrl(i) && UFix(i) > last_grant || ctrl(i + n)) + + var choose = Bits(n-1) + for (i <- n-2 to 0 by -1) + choose = Mux(io.in(i).meta.valid, Bits(i), choose) + for (i <- n-1 to 1 by -1) + choose = Mux(io.in(i).meta.valid && UFix(i) > last_grant, Bits(i), choose) + meta_chosen := choose + + when (io.out.meta.fire()) { last_grant := meta_chosen } +} + case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { From 0672773c1a762db43ad66cf6e3e70f2bff6e061f Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 9 May 2013 02:14:44 -0700 Subject: [PATCH 180/374] for now, don't use asserts outside of components --- uncore/src/coherence.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index ec932ec9..42b3495e 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -842,7 +842,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { - Assert( incoming.p_type === probeInvalidateOthers && needsWriteback(state), "Bad probe request type, should be impossible.") + //Assert( incoming.p_type === probeInvalidateOthers && needsWriteback(state), "Bad probe request type, should be impossible.") val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> Mux(uFixListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), releaseInvalidateDataMigratory, releaseInvalidateData), From 4c1f105ce94eb9dd3ae7902272f5301119278c15 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 21 May 2013 17:19:07 -0700 Subject: [PATCH 181/374] added PairedData link type with matching crossbar, ported tilelink and uncore to use --- uncore/src/coherence.scala | 6 +- uncore/src/package.scala | 50 +++++++++-- uncore/src/tilelink.scala | 93 ++++++++------------ uncore/src/uncore.scala | 168 ++++++++++++++++++------------------- 4 files changed, 158 insertions(+), 159 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index ec932ec9..6a1eaf80 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -3,9 +3,9 @@ package uncore import Chisel._ import Constants._ -trait CoherenceAgentRole -trait ClientCoherenceAgent extends CoherenceAgentRole -trait MasterCoherenceAgent extends CoherenceAgentRole +abstract trait CoherenceAgentRole +abstract trait ClientCoherenceAgent extends CoherenceAgentRole +abstract trait MasterCoherenceAgent extends CoherenceAgentRole abstract class CoherencePolicy { def isHit (cmd: Bits, state: UFix): Bool diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 46c39d51..754621ad 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -9,8 +9,9 @@ import scala.collection.mutable.Stack implicit def toOption[A](a: A) = Option(a) class PairedDataIO[M <: Data, D <: Data]()(m: => M, d: => D) extends Bundle { - val meta = (new FIFOIO()){ m } - val data = (new FIFOIO()){ d } + val meta = new FIFOIO()(m) + val data = new FIFOIO()(d) + override def clone = { new PairedDataIO()(m,d).asInstanceOf[this.type] } } class PairedArbiterIO[M <: Data, D <: Data](n: Int)(m: => M, d: => D) extends Bundle { @@ -18,9 +19,10 @@ class PairedArbiterIO[M <: Data, D <: Data](n: Int)(m: => M, d: => D) extends Bu val out = new PairedDataIO()(m,d) val meta_chosen = Bits(OUTPUT, log2Up(n)) val data_chosen = Bits(OUTPUT, log2Up(n)) + override def clone = { new PairedArbiterIO(n)(m,d).asInstanceOf[this.type] } } -class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock: Option[M => Bool] = None)(meta: => M, data: => D) { +class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock: Option[M => Bool] = None)(meta: => M, data: => D) extends Component { require(isPow2(count)) val io = new PairedArbiterIO(n)(meta,data) val locked = if(count > 1) Reg(resetVal = Bool(false)) else Bool(false) @@ -28,11 +30,13 @@ class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock val grant = List.fill(n)(Bool()) val meta_chosen = Bits(width = log2Up(n)) - val chosen_meta_has_data = io.out.meta.valid && needsLock.map(_(io.out.meta.bits)).getOrElse(Bool(true)) - (0 until n).map(i => io.in(i).meta.ready := grant(i) && io.out.meta.ready) - (0 until n).map(i => io.in(i).data.ready := Mux(locked, lockIdx === UFix(i), grant(i) && chosen_meta_has_data) && io.out.data.ready) - io.out.meta.valid := io.in(meta_chosen).meta.valid - io.out.data.valid := Mux(locked, io.in(lockIdx).data.valid, io.in(meta_chosen).data.valid && chosen_meta_has_data) + val chosen_meta_has_data = needsLock.map(_(io.in(meta_chosen).meta.bits)).getOrElse(Bool(true)) + val valid_meta_has_data = io.in(meta_chosen).meta.valid && chosen_meta_has_data + val grant_chosen_meta = !(locked && chosen_meta_has_data) + (0 until n).map(i => io.in(i).meta.ready := grant(i) && grant_chosen_meta && io.out.meta.ready) + (0 until n).map(i => io.in(i).data.ready := Mux(locked, lockIdx === UFix(i), grant(i) && valid_meta_has_data) && io.out.data.ready) + io.out.meta.valid := io.in(meta_chosen).meta.valid && grant_chosen_meta + io.out.data.valid := Mux(locked, io.in(lockIdx).data.valid, io.in(meta_chosen).data.valid && valid_meta_has_data) io.out.meta.bits := io.in(meta_chosen).meta.bits io.out.data.bits := Mux(locked, io.in(lockIdx).data.bits, io.in(meta_chosen).data.bits) io.meta_chosen := meta_chosen @@ -70,6 +74,34 @@ class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock when (io.out.meta.fire()) { last_grant := meta_chosen } } +class PairedCrossbar[M <: Data, D <: Data](count: Int, needsLock: Option[BasicCrossbarIO[M] => Bool] = None)(meta: => M, data: => D)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { + val io = new Bundle { + val in = Vec(conf.nEndpoints){new PairedDataIO()(new BasicCrossbarIO()(meta),new BasicCrossbarIO()(data))}.flip + val out = Vec(conf.nEndpoints){new PairedDataIO()(new BasicCrossbarIO()(meta),new BasicCrossbarIO()(data))} + } + + val metaRdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) + val dataRdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) + val rdyVecs = metaRdyVecs zip dataRdyVecs + + io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { + val rrarb = new PairedLockingRRArbiter(conf.nEndpoints, count, needsLock)(io.in(0).meta.bits.clone, io.in(0).data.bits.clone) + rrarb.io.in zip io.in zip rdys._1 zip rdys._2 map { case (((arb, in), meta_rdy), data_rdy) => { + arb.meta.valid := in.meta.valid && (in.meta.bits.header.dst === UFix(i)) + arb.meta.bits := in.meta.bits + meta_rdy := arb.meta.ready && (in.meta.bits.header.dst === UFix(i)) + arb.data.valid := in.data.valid && (in.data.bits.header.dst === UFix(i)) + arb.data.bits := in.data.bits + data_rdy := arb.data.ready && (in.data.bits.header.dst === UFix(i)) + }} + out <> rrarb.io.out + }} + for(i <- 0 until conf.nEndpoints) { + io.in(i).meta.ready := rdyVecs.map(r => r._1(i)).reduceLeft(_||_) + io.in(i).data.ready := rdyVecs.map(r => r._2(i)).reduceLeft(_||_) + } +} + case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { @@ -88,7 +120,7 @@ class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkCon abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Component -class BasicCrossbar[T <: Data](count: Int)(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { +class BasicCrossbar[T <: Data](count: Int = 1)(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { val io = new Bundle { val in = Vec(conf.nEndpoints){(new FIFOIO){(new BasicCrossbarIO){data}}}.flip val out = Vec(conf.nEndpoints){(new FIFOIO){(new BasicCrossbarIO){data}}} diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index c7f82da2..589254f9 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -116,82 +116,55 @@ class Grant extends MasterSourcedMessage with HasMemData with HasClientTransacti class GrantAck extends ClientSourcedMessage with HasMasterTransactionId -abstract class DirectionalFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) -class ClientSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) -class ClientSourcedDataIO[T <: Data]()(data: => T) extends ClientSourcedIO()(data) -class MasterSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) {flip()} +trait DirectionalIO +trait ClientSourcedIO extends DirectionalIO +trait MasterSourcedIO extends DirectionalIO +class ClientSourcedFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) with ClientSourcedIO { + override def clone = { new ClientSourcedFIFOIO()(data).asInstanceOf[this.type] } +} +class ClientSourcedDataIO[M <: Data, D <: Data]()(meta: => M, data: => D) extends PairedDataIO()(meta,data) with ClientSourcedIO { + override def clone = { new ClientSourcedDataIO()(meta,data).asInstanceOf[this.type] } +} +class MasterSourcedFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) with MasterSourcedIO { + flip() + override def clone = { new MasterSourcedFIFOIO()(data).asInstanceOf[this.type] } +} +class MasterSourcedDataIO[M <: Data, D <: Data]()(meta: => M, data: => D) extends PairedDataIO()(meta,data) with MasterSourcedIO { + flip() + override def clone = { new MasterSourcedDataIO()(meta,data).asInstanceOf[this.type] } +} class UncachedTileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val acquire = (new ClientSourcedIO){(new LogicalNetworkIO){new Acquire }} - val acquire_data = (new ClientSourcedDataIO){(new LogicalNetworkIO){new AcquireData }} - val grant = (new MasterSourcedIO) {(new LogicalNetworkIO){new Grant }} - val grant_ack = (new ClientSourcedIO){(new LogicalNetworkIO){new GrantAck }} + val acquire = new ClientSourcedDataIO()(new LogicalNetworkIO()(new Acquire), new LogicalNetworkIO()(new AcquireData)) + val grant = new MasterSourcedFIFOIO()(new LogicalNetworkIO()(new Grant)) + val grant_ack = new ClientSourcedFIFOIO()(new LogicalNetworkIO()(new GrantAck)) override def clone = { new UncachedTileLinkIO().asInstanceOf[this.type] } } class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends UncachedTileLinkIO()(conf) { - val probe = (new MasterSourcedIO){(new LogicalNetworkIO){new Probe }} - val release = (new ClientSourcedIO){(new LogicalNetworkIO){new Release }} - val release_data = (new ClientSourcedDataIO){(new LogicalNetworkIO){new ReleaseData }} + val probe = new MasterSourcedFIFOIO()(new LogicalNetworkIO()(new Probe)) + val release = new ClientSourcedDataIO()(new LogicalNetworkIO()(new Release), new LogicalNetworkIO()(new ReleaseData)) override def clone = { new TileLinkIO().asInstanceOf[this.type] } } -object UncachedTileLinkIOArbiterShim { - def apply[T <: HasClientTransactionId](in: ClientSourcedIO[LogicalNetworkIO[T]], id: Int, max: Int)(implicit lconf: LogicalNetworkConfiguration) = { - val shim = (new UncachedTileLinkIOArbiterShim(id, max)){in.bits.payload.clone} - shim.io.in <> in - shim.io.out - } -} -class UncachedTileLinkIOArbiterShim[T <: HasClientTransactionId](id: Int, max: Int)(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { - val io = new Bundle { - val in = (new ClientSourcedIO){(new LogicalNetworkIO){ data }}.flip - val out = (new ClientSourcedIO){(new LogicalNetworkIO){ data }} - } - io.out.bits := io.in.bits - io.out.bits.payload.client_xact_id := Cat(io.in.bits.payload.client_xact_id, UFix(id, log2Up(max))) - io.out.valid := io.in.valid - io.in.ready := io.out.ready -} - - -class UncachedTileLinkIOArbiter(n: Int)(implicit conf: LogicalNetworkConfiguration) extends Component { +class UncachedTileLinkIOArbiter(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Component { val io = new Bundle { val in = Vec(n) { new UncachedTileLinkIO }.flip val out = new UncachedTileLinkIO } - val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) - val mem_cnt_next = mem_cnt + UFix(1) - val locked = Reg(resetVal = Bool(false)) - val lock_idx = Reg(resetVal = UFix(n)) - - when(io.out.acquire_data.valid && io.out.acquire_data.ready) { - mem_cnt := mem_cnt_next - when(!locked) { - locked := Bool(true) - lock_idx := Vec(io.in.map{ in => in.acquire_data.ready && in.acquire_data.valid}){Bool()}.indexWhere{i: Bool => i} - } - when(mem_cnt_next === UFix(0)) { - locked := Bool(false) - } - } - - val acqd_grant = ArbiterCtrl(io.in.map(_.acquire_data.valid)) - (0 until n).map(i => io.in(i).acquire_data.ready := Mux(locked, UFix(i) === lock_idx, acqd_grant(i)) && io.out.acquire_data.ready) - var acqd_bits = io.in(n-1).acquire_data.bits - for (i <- n-2 to 0 by -1) { - acqd_bits = Mux(io.in(i).acquire_data.valid, io.in(i).acquire_data.bits, acqd_bits) - } - val locked_req = io.in(lock_idx).acquire_data - io.out.acquire_data.bits := Mux(locked, locked_req.bits, acqd_bits) - io.out.acquire_data.valid := Mux(locked, locked_req.valid, io.in.map(_.acquire_data.valid).reduce(_||_)) - - val acq_arb = (new Arbiter(n)){ (new LogicalNetworkIO){new Acquire} } + def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) + val acq_arb = new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData}) io.out.acquire <> acq_arb.io.out - io.in.map(_.acquire).zipWithIndex.map{ case(acq, id) => UncachedTileLinkIOArbiterShim(acq, id, n) }.zip(acq_arb.io.in).map{ case (req, arb) => req <> arb} + io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { + arb.data <> req.data + arb.meta.valid := req.meta.valid + arb.meta.bits := req.meta.bits + arb.meta.bits.payload.client_xact_id := Cat(req.meta.bits.payload.client_xact_id, UFix(id, log2Up(n))) + req.meta.ready := arb.meta.ready + }} - val grant_ack_arb = (new Arbiter(n)){ (new LogicalNetworkIO){new GrantAck} } + val grant_ack_arb = (new RRArbiter(n)){ (new LogicalNetworkIO){new GrantAck} } io.out.grant_ack <> grant_ack_arb.io.out grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index f0571d89..3a3c36ae 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -31,23 +31,22 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends // Handle acquire transaction initiation val acquire = io.client.acquire - val acquire_data = io.client.acquire_data val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val block_acquires = any_acquire_conflict val alloc_arb = (new Arbiter(trackerList.size)) { Bool() } for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.client - alloc_arb.io.in(i).valid := t.acquire.ready - t.acquire.bits := acquire.bits - t.acquire.valid := alloc_arb.io.in(i).ready + alloc_arb.io.in(i).valid := t.acquire.meta.ready + t.acquire.meta.bits := acquire.meta.bits + t.acquire.meta.valid := alloc_arb.io.in(i).ready - t.acquire_data.bits := acquire_data.bits - t.acquire_data.valid := acquire_data.valid + t.acquire.data.bits := acquire.data.bits + t.acquire.data.valid := acquire.data.valid } - acquire.ready := trackerList.map(_.io.client.acquire.ready).reduce(_||_) && !block_acquires - acquire_data.ready := trackerList.map(_.io.client.acquire_data.ready).reduce(_||_) - alloc_arb.io.out.ready := acquire.valid && !block_acquires + acquire.meta.ready := trackerList.map(_.io.client.acquire.meta.ready).reduce(_||_) && !block_acquires + acquire.data.ready := trackerList.map(_.io.client.acquire.data.ready).reduce(_||_) + alloc_arb.io.out.ready := acquire.meta.valid && !block_acquires // Handle probe request generation val probe_arb = (new Arbiter(trackerList.size)){(new LogicalNetworkIO){ new Probe }} @@ -56,22 +55,21 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends // Handle releases, which might be voluntary and might have data val release = io.client.release - val release_data = io.client.release_data - val voluntary = co.isVoluntary(release.bits.payload) + val voluntary = co.isVoluntary(release.meta.bits.payload) val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) val block_releases = Bool(false) val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)){Bool()}.lastIndexWhere{b: Bool => b} //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UFix(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response - val release_idx = Mux(voluntary, release.bits.header.src, release.bits.payload.master_xact_id) + val release_idx = Mux(voluntary, release.meta.bits.header.src, release.meta.bits.payload.master_xact_id) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.client - t.release.bits := release.bits - t.release.valid := release.valid && (release_idx === UFix(i)) && !block_releases - t.release_data.bits := release_data.bits - t.release_data.valid := release_data.valid + t.release.meta.bits := release.meta.bits + t.release.meta.valid := release.meta.valid && (release_idx === UFix(i)) && !block_releases + t.release.data.bits := release.data.bits + t.release.data.valid := release.data.valid } - release.ready := Vec(trackerList.map(_.io.client.release.ready)){Bool()}(release_idx) && !block_releases - release_data.ready := trackerList.map(_.io.client.release_data.ready).reduce(_||_) + release.meta.ready := Vec(trackerList.map(_.io.client.release.meta.ready)){Bool()}(release_idx) && !block_releases + release.data.ready := trackerList.map(_.io.client.release.data.ready).reduce(_||_) // Reply to initial requestor val grant_arb = (new Arbiter(trackerList.size)){(new LogicalNetworkIO){ new Grant }} @@ -85,7 +83,7 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends ack.ready := Bool(true) // Create an arbiter for the one memory port - val outer_arb = new UncachedTileLinkIOArbiter(trackerList.size) + val outer_arb = new UncachedTileLinkIOArbiter(trackerList.size, conf.co) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.master } io.master <> outer_arb.io.out } @@ -114,22 +112,22 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: Uncore val cmd_to_read = co.getUncachedReadAcquire(xact.addr, UFix(trackerId)) io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.bits.payload.addr) && (state != s_idle) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.meta.bits.payload.addr) && (state != s_idle) io.master.grant.ready := Bool(false) - io.master.acquire.valid := Bool(false) - io.master.acquire.bits.payload := cmd_to_write + io.master.acquire.meta.valid := Bool(false) + io.master.acquire.meta.bits.payload := cmd_to_write //TODO io.master.acquire.bits.header.dst - io.master.acquire.bits.header.src := UFix(bankId) - io.master.acquire_data.valid := Bool(false) - io.master.acquire_data.bits.payload.data := UFix(0) + io.master.acquire.meta.bits.header.src := UFix(bankId) + io.master.acquire.data.valid := Bool(false) + io.master.acquire.data.bits.payload.data := UFix(0) //TODO io.master.acquire_data.bits.header.dst - io.master.acquire_data.bits.header.src := UFix(bankId) - io.client.acquire.ready := Bool(false) - io.client.acquire_data.ready := Bool(false) + io.master.acquire.data.bits.header.src := UFix(bankId) + io.client.acquire.meta.ready := Bool(false) + io.client.acquire.data.ready := Bool(false) io.client.probe.valid := Bool(false) - io.client.release.ready := Bool(false) - io.client.release_data.ready := Bool(false) // DNC + io.client.release.meta.ready := Bool(false) + io.client.release.data.ready := Bool(false) // DNC io.client.grant.valid := Bool(false) io.client.grant.bits.payload.g_type := co.getGrantType(xact, UFix(0)) io.client.grant.bits.payload.client_xact_id := xact.client_xact_id @@ -141,11 +139,11 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: Uncore switch (state) { is(s_idle) { - io.client.release.ready := Bool(true) - when( io.client.release.valid ) { - xact := io.client.release.bits.payload - init_client_id_ := io.client.release.bits.header.src - release_data_needs_write := co.messageHasData(io.client.release.bits.payload) + io.client.release.meta.ready := Bool(true) + when( io.client.release.meta.valid ) { + xact := io.client.release.meta.bits.payload + init_client_id_ := io.client.release.meta.bits.header.src + release_data_needs_write := co.messageHasData(io.client.release.meta.bits.payload) mem_cnt := UFix(0) mem_cmd_sent := Bool(false) state := s_mem @@ -153,10 +151,9 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: Uncore } is(s_mem) { when (release_data_needs_write) { - doOuterReqWrite(io.master.acquire, - io.master.acquire_data, - io.client.release_data, - release_data_needs_write, + doOuterReqWrite(io.master.acquire, + io.client.release.data, + release_data_needs_write, mem_cmd_sent, init_client_id_) } . otherwise { state := s_ack } @@ -192,20 +189,20 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura if (conf.ln.nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence val probe_self = Bool(true) //co.needsSelfProbe(io.client.acquire.bits.payload) - val myflag = Mux(probe_self, Bits(0), UFixToOH(io.client.acquire.bits.header.src(log2Up(conf.ln.nClients)-1,0))) + val myflag = Mux(probe_self, Bits(0), UFixToOH(io.client.acquire.meta.bits.header.src(log2Up(conf.ln.nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } - io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, io.client.acquire.bits.payload.addr) && (state != s_idle) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.bits.payload.addr) && (state != s_idle) - io.master.acquire.valid := Bool(false) - io.master.acquire.bits.payload := co.getUncachedReadAcquire(xact.addr, UFix(trackerId)) + io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, io.client.acquire.meta.bits.payload.addr) && (state != s_idle) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.meta.bits.payload.addr) && (state != s_idle) + io.master.acquire.meta.valid := Bool(false) + io.master.acquire.meta.bits.payload := co.getUncachedReadAcquire(xact.addr, UFix(trackerId)) //TODO io.master.acquire.bits.header.dst - io.master.acquire.bits.header.src := UFix(bankId) - io.master.acquire_data.valid := Bool(false) - io.master.acquire_data.bits.payload.data := UFix(0) + io.master.acquire.meta.bits.header.src := UFix(bankId) + io.master.acquire.data.valid := Bool(false) + io.master.acquire.data.bits.payload.data := UFix(0) //TODO io.master.acquire_data.bits.header.dst - io.master.acquire_data.bits.header := UFix(bankId) + io.master.acquire.data.bits.header := UFix(bankId) io.client.probe.valid := Bool(false) io.client.probe.bits.payload.p_type := co.getProbeType(xact.a_type, UFix(0)) io.client.probe.bits.payload.master_xact_id := UFix(trackerId) @@ -219,22 +216,22 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura io.client.grant.bits.header.dst := init_client_id_ io.client.grant.bits.header.src := UFix(bankId) io.client.grant.valid := (io.master.grant.valid && (UFix(trackerId) === io.master.grant.bits.payload.client_xact_id)) - io.client.acquire.ready := Bool(false) - io.client.acquire_data.ready := Bool(false) - io.client.release.ready := Bool(false) - io.client.release_data.ready := Bool(false) + io.client.acquire.meta.ready := Bool(false) + io.client.acquire.data.ready := Bool(false) + io.client.release.meta.ready := Bool(false) + io.client.release.data.ready := Bool(false) io.master.grant.ready := io.client.grant.ready io.client.grant_ack.valid := Bool(false) switch (state) { is(s_idle) { - io.client.acquire.ready := Bool(true) - when( io.client.acquire.valid ) { - xact := io.client.acquire.bits.payload - init_client_id_ := io.client.acquire.bits.header.src + io.client.acquire.meta.ready := Bool(true) + when( io.client.acquire.meta.valid ) { + xact := io.client.acquire.meta.bits.payload + init_client_id_ := io.client.acquire.meta.bits.header.src init_sharer_cnt_ := UFix(conf.ln.nClients) // TODO: Broadcast only - acquire_data_needs_write := co.messageHasData(io.client.acquire.bits.payload) - x_needs_read := co.needsOuterRead(io.client.acquire.bits.payload.a_type, UFix(0)) + acquire_data_needs_write := co.messageHasData(io.client.acquire.meta.bits.payload) + x_needs_read := co.needsOuterRead(io.client.acquire.meta.bits.payload.a_type, UFix(0)) probe_flags := probe_initial_flags mem_cnt := UFix(0) r_w_mem_cmd_sent := Bool(false) @@ -253,21 +250,20 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura when(io.client.probe.ready) { probe_flags := probe_flags & ~(UFixToOH(curr_p_id)) } - io.client.release.ready := Bool(true) - when(io.client.release.valid) { + io.client.release.meta.ready := Bool(true) + when(io.client.release.meta.valid) { if(conf.ln.nClients > 1) release_count := release_count - UFix(1) when(release_count === UFix(1)) { state := s_mem } - when( co.messageHasData(io.client.release.bits.payload)) { + when( co.messageHasData(io.client.release.meta.bits.payload)) { release_data_needs_write := Bool(true) - release_data_client_id := io.client.release.bits.header.src + release_data_client_id := io.client.release.meta.bits.header.src } } when (release_data_needs_write) { - doOuterReqWrite(io.master.acquire, - io.master.acquire_data, - io.client.release_data, + doOuterReqWrite(io.master.acquire, + io.client.release.data, release_data_needs_write, r_w_mem_cmd_sent, release_data_client_id) @@ -275,18 +271,16 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura } is(s_mem) { when (release_data_needs_write) { - doOuterReqWrite(io.master.acquire, - io.master.acquire_data, - io.client.release_data, - release_data_needs_write, - r_w_mem_cmd_sent, + doOuterReqWrite(io.master.acquire, + io.client.release.data, + release_data_needs_write, + r_w_mem_cmd_sent, release_data_client_id) } . elsewhen(acquire_data_needs_write) { - doOuterReqWrite(io.master.acquire, - io.master.acquire_data, - io.client.acquire_data, - acquire_data_needs_write, - a_w_mem_cmd_sent, + doOuterReqWrite(io.master.acquire, + io.client.acquire.data, + acquire_data_needs_write, + a_w_mem_cmd_sent, init_client_id_) } . elsewhen (x_needs_read) { doOuterReqRead(io.master.acquire, x_needs_read) @@ -313,18 +307,18 @@ abstract trait OuterRequestGenerator { val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) - def doOuterReqWrite[T <: Data](master_acq: FIFOIO[LogicalNetworkIO[Acquire]], master_acq_data: FIFOIO[LogicalNetworkIO[AcquireData]], client_data: FIFOIO[LogicalNetworkIO[T]], trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UFix) { + def doOuterReqWrite[T <: HasMemData](master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], client_data: FIFOIO[LogicalNetworkIO[T]], trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UFix) { val do_write = client_data.valid && (client_data.bits.header.src === desired_client_data_src_id) - master_acq.bits.payload := cmd_to_write - master_acq_data.bits.payload := client_data.bits.payload - when(master_acq.ready && master_acq.valid) { + master_acq.meta.bits.payload := cmd_to_write + master_acq.data.bits.payload := client_data.bits.payload + when(master_acq.meta.fire()) { cmd_sent := Bool(true) } when (do_write) { - master_acq.valid := !cmd_sent && master_acq_data.ready - when (master_acq.ready || cmd_sent) { - master_acq_data.valid := client_data.valid - when(master_acq_data.ready) { + master_acq.meta.valid := !cmd_sent + when (master_acq.meta.ready || cmd_sent) { + master_acq.data.valid := client_data.valid + when(master_acq.data.ready) { client_data.ready:= Bool(true) mem_cnt := mem_cnt_next when(mem_cnt === UFix(REFILL_CYCLES-1)) { @@ -335,10 +329,10 @@ abstract trait OuterRequestGenerator { } } - def doOuterReqRead(master_acq: FIFOIO[LogicalNetworkIO[Acquire]], trigger: Bool) { - master_acq.valid := Bool(true) - master_acq.bits.payload := cmd_to_read - when(master_acq.ready) { + def doOuterReqRead(master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], trigger: Bool) { + master_acq.meta.valid := Bool(true) + master_acq.meta.bits.payload := cmd_to_read + when(master_acq.meta.ready) { trigger := Bool(false) } } From cf02f1ef014ab7f644f1657ecd97f37152cda36f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 22 Apr 2013 16:48:55 -0700 Subject: [PATCH 182/374] use new locking round robin arbiter --- uncore/src/package.scala | 51 ++-------------------------------------- 1 file changed, 2 insertions(+), 49 deletions(-) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 4133a9d3..7e1fc7a0 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -6,6 +6,8 @@ import scala.collection.mutable.Stack //TODO: Remove these Networking classes from the package object once Scala bug //SI-3439 is resolved. +implicit def toOption[A](a: A) = Option(a) + case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { @@ -24,55 +26,6 @@ class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkCon abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Component -class LockingRRArbiter[T <: Data](n: Int, count: Int)(data: => T) extends Component { - val io = new ioArbiter(n)(data) - - val cnt = Reg(resetVal = UFix(0, width = log2Up(count))) - val cnt_next = cnt + UFix(1) - val locked = Reg(resetVal = Bool(false)) - val lock_idx = Reg(resetVal = UFix(n)) - - if(count > 1){ - when(io.out.valid && io.out.ready) { - cnt := cnt_next - when(!locked) { - locked := Bool(true) - lock_idx := Vec(io.in.map{ in => in.ready && in.valid}){Bool()}.indexWhere{i: Bool => i} - } - when(cnt_next === UFix(0)) { - locked := Bool(false) - } - } - } else { - locked := Bool(false) - lock_idx := UFix(n) - cnt := UFix(0) - } - - - val last_grant = Reg(resetVal = Bits(0, log2Up(n))) - val g = ArbiterCtrl((0 until n).map(i => io.in(i).valid && UFix(i) > last_grant) ++ io.in.map(_.valid)) - val grant = (0 until n).map(i => g(i) && UFix(i) > last_grant || g(i+n)) - (0 until n).map(i => io.in(i).ready := Mux(locked, lock_idx === UFix(i), grant(i)) && io.out.ready) - - var choose = Bits(n-1) - for (i <- n-2 to 0 by -1) - choose = Mux(io.in(i).valid, Bits(i), choose) - for (i <- n-1 to 1 by -1) - choose = Mux(io.in(i).valid && UFix(i) > last_grant, Bits(i), choose) - choose = Mux(locked, lock_idx, choose) - when (io.out.valid && io.out.ready) { - last_grant := choose - } - - val dvec = Vec(n) { data } - (0 until n).map(i => dvec(i) := io.in(i).bits ) - - io.out.valid := Mux(locked, io.in(lock_idx).valid, io.in.map(_.valid).foldLeft(Bool(false))( _ || _)) - io.out.bits := dvec(choose) - io.chosen := choose -} - class BasicCrossbar[T <: Data](count: Int)(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { val io = new Bundle { val in = Vec(conf.nEndpoints){(new FIFOIO){(new BasicCrossbarIO){data}}}.flip From db8e5fda9b908e908f78110012836dbab5b80c72 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 9 Jul 2013 15:37:42 -0700 Subject: [PATCH 183/374] new tilelink arbiter types, reduced release xact trackers --- uncore/src/consts.scala | 2 +- uncore/src/tilelink.scala | 96 ++++++++++++++++++++++++++++++++++++--- uncore/src/uncore.scala | 6 +-- 3 files changed, 94 insertions(+), 10 deletions(-) diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index 9d343b3f..aaa7b7a8 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -11,7 +11,7 @@ abstract trait CoherenceConfigConstants { trait UncoreConstants { val NGLOBAL_ACQ_XACTS = 8 - val NGLOBAL_REL_XACTS = 4 + val NGLOBAL_REL_XACTS = 1 val MASTER_XACT_ID_MAX_BITS = log2Up(NGLOBAL_ACQ_XACTS+NGLOBAL_REL_XACTS) val CACHE_DATA_SIZE_IN_BYTES = 1 << 6 } diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index 589254f9..2e595c34 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -147,12 +147,24 @@ class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends UncachedTil override def clone = { new TileLinkIO().asInstanceOf[this.type] } } -class UncachedTileLinkIOArbiter(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Component { +/* + * TODO: Merge the below classes into children of an abstract class in Chisel 2.0 +abstract class UncachedTileLinkIOArbiter(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Component { + def acquireClientXactId(in: Acquire, id: Int): Bits + def grantClientXactId(in: Grant): Bits + def arbIdx(in: Grant): UFix +} +*/ + +class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Component { + def acquireClientXactId(in: Acquire, id: Int) = Cat(in.client_xact_id, UFix(id, log2Up(n))) + def grantClientXactId(in: Grant) = in.client_xact_id >> UFix(log2Up(n)) + def arbIdx(in: Grant) = in.client_xact_id(log2Up(n)-1,0).toUFix + val io = new Bundle { val in = Vec(n) { new UncachedTileLinkIO }.flip val out = new UncachedTileLinkIO } - def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) val acq_arb = new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData}) io.out.acquire <> acq_arb.io.out @@ -160,7 +172,7 @@ class UncachedTileLinkIOArbiter(n: Int, co: CoherencePolicy)(implicit conf: Logi arb.data <> req.data arb.meta.valid := req.meta.valid arb.meta.bits := req.meta.bits - arb.meta.bits.payload.client_xact_id := Cat(req.meta.bits.payload.client_xact_id, UFix(id, log2Up(n))) + arb.meta.bits.payload.client_xact_id := acquireClientXactId(req.meta.bits.payload, id) req.meta.ready := arb.meta.ready }} @@ -170,13 +182,85 @@ class UncachedTileLinkIOArbiter(n: Int, co: CoherencePolicy)(implicit conf: Logi io.out.grant.ready := Bool(false) for (i <- 0 until n) { - val tag = io.out.grant.bits.payload.client_xact_id io.in(i).grant.valid := Bool(false) - when (tag(log2Up(n)-1,0) === UFix(i)) { + when (arbIdx(io.out.grant.bits.payload) === UFix(i)) { io.in(i).grant.valid := io.out.grant.valid io.out.grant.ready := io.in(i).grant.ready } io.in(i).grant.bits := io.out.grant.bits - io.in(i).grant.bits.payload.client_xact_id := tag >> UFix(log2Up(n)) + io.in(i).grant.bits.payload.client_xact_id := grantClientXactId(io.out.grant.bits.payload) } } + +class UncachedTileLinkIOArbiterThatPassesId(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Component { + def acquireClientXactId(in: Acquire, id: Int) = in.client_xact_id + def grantClientXactId(in: Grant) = in.client_xact_id + def arbIdx(in: Grant): UFix = in.client_xact_id + + val io = new Bundle { + val in = Vec(n) { new UncachedTileLinkIO }.flip + val out = new UncachedTileLinkIO + } + def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) + val acq_arb = new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData}) + io.out.acquire <> acq_arb.io.out + io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { + arb.data <> req.data + arb.meta.valid := req.meta.valid + arb.meta.bits := req.meta.bits + arb.meta.bits.payload.client_xact_id := acquireClientXactId(req.meta.bits.payload, id) + req.meta.ready := arb.meta.ready + }} + + val grant_ack_arb = (new RRArbiter(n)){ (new LogicalNetworkIO){new GrantAck} } + io.out.grant_ack <> grant_ack_arb.io.out + grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } + + io.out.grant.ready := Bool(false) + for (i <- 0 until n) { + io.in(i).grant.valid := Bool(false) + when (arbIdx(io.out.grant.bits.payload) === UFix(i)) { + io.in(i).grant.valid := io.out.grant.valid + io.out.grant.ready := io.in(i).grant.ready + } + io.in(i).grant.bits := io.out.grant.bits + io.in(i).grant.bits.payload.client_xact_id := grantClientXactId(io.out.grant.bits.payload) + } +} + +class UncachedTileLinkIOArbiterThatUsesNewId(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Component { + def acquireClientXactId(in: Acquire, id: Int) = UFix(id, log2Up(n)) + def grantClientXactId(in: Grant) = UFix(0) // DNC + def arbIdx(in: Grant) = in.client_xact_id + + val io = new Bundle { + val in = Vec(n) { new UncachedTileLinkIO }.flip + val out = new UncachedTileLinkIO + } + def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) + val acq_arb = new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData}) + io.out.acquire <> acq_arb.io.out + io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { + arb.data <> req.data + arb.meta.valid := req.meta.valid + arb.meta.bits := req.meta.bits + arb.meta.bits.payload.client_xact_id := acquireClientXactId(req.meta.bits.payload, id) + req.meta.ready := arb.meta.ready + }} + + val grant_ack_arb = (new RRArbiter(n)){ (new LogicalNetworkIO){new GrantAck} } + io.out.grant_ack <> grant_ack_arb.io.out + grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } + + io.out.grant.ready := Bool(false) + for (i <- 0 until n) { + io.in(i).grant.valid := Bool(false) + when (arbIdx(io.out.grant.bits.payload) === UFix(i)) { + io.in(i).grant.valid := io.out.grant.valid + io.out.grant.ready := io.in(i).grant.ready + } + io.in(i).grant.bits := io.out.grant.bits + io.in(i).grant.bits.payload.client_xact_id := grantClientXactId(io.out.grant.bits.payload) + } +} + diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 3a3c36ae..9d0de31b 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -23,7 +23,7 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends { implicit val lnConf = conf.ln val co = conf.co - require(conf.ln.nClients < NGLOBAL_REL_XACTS) //TODO: handle in config + //require(conf.ln.nClients < NGLOBAL_REL_XACTS) //TODO: handle in config val trackerList = (0 until NGLOBAL_REL_XACTS).map(new VoluntaryReleaseTracker(_, bankId)) ++ (NGLOBAL_REL_XACTS until NGLOBAL_REL_XACTS + NGLOBAL_ACQ_XACTS).map(new AcquireTracker(_, bankId)) @@ -60,7 +60,7 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends val block_releases = Bool(false) val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)){Bool()}.lastIndexWhere{b: Bool => b} //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UFix(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response - val release_idx = Mux(voluntary, release.meta.bits.header.src, release.meta.bits.payload.master_xact_id) + val release_idx = Mux(voluntary, UFix(0), release.meta.bits.payload.master_xact_id) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.client t.release.meta.bits := release.meta.bits @@ -83,7 +83,7 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends ack.ready := Bool(true) // Create an arbiter for the one memory port - val outer_arb = new UncachedTileLinkIOArbiter(trackerList.size, conf.co) + val outer_arb = new UncachedTileLinkIOArbiterThatPassesId(trackerList.size, conf.co) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.master } io.master <> outer_arb.io.out } From d8440b042a4d74c3f5f437ab73b328c5ff1ad94c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 24 Jul 2013 23:22:36 -0700 Subject: [PATCH 184/374] Make compatible with scala 2.10. Refactor constants into package object. Remove networking primitives from package object. Clean up request generators. Chnage ++ to +: for appending to io.incoherent. --- uncore/src/coherence.scala | 2 - uncore/src/consts.scala | 2 + uncore/src/llc.scala | 3 - uncore/src/memserdes.scala | 3 - uncore/src/network.scala | 193 ++++++++++++++++++++++++++++++++++ uncore/src/package.scala | 210 ++----------------------------------- uncore/src/slowio.scala | 2 - uncore/src/temp.scala | 13 --- uncore/src/tilelink.scala | 2 - uncore/src/uncore.scala | 19 ++-- 10 files changed, 211 insertions(+), 238 deletions(-) create mode 100644 uncore/src/network.scala delete mode 100644 uncore/src/temp.scala diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index 03c5c8f1..b869436a 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -1,7 +1,5 @@ package uncore - import Chisel._ -import Constants._ abstract trait CoherenceAgentRole abstract trait ClientCoherenceAgent extends CoherenceAgentRole diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index aaa7b7a8..4e3c814e 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -36,6 +36,7 @@ trait TileLinkSizeConstants extends val ACQUIRE_ATOMIC_OP_BITS = 4 } +object MemoryOpConstants extends MemoryOpConstants trait MemoryOpConstants { val MT_X = Bits("b???", 3); val MT_B = Bits("b000", 3); @@ -82,6 +83,7 @@ trait MemoryInterfaceConstants extends val REFILL_CYCLES = CACHE_DATA_SIZE_IN_BYTES*8/MEM_DATA_BITS } +object AddressConstants extends AddressConstants trait AddressConstants { val PADDR_BITS = 32 val VADDR_BITS = 43; diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index 1f9f4b2a..6b0c77f2 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -1,8 +1,5 @@ package uncore - import Chisel._ -import Node._ -import Constants._ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component { diff --git a/uncore/src/memserdes.scala b/uncore/src/memserdes.scala index 88a6f1b3..c7c8e773 100644 --- a/uncore/src/memserdes.scala +++ b/uncore/src/memserdes.scala @@ -1,8 +1,5 @@ package uncore - import Chisel._ -import Node._ -import Constants._ import scala.math._ class ioMemSerialized(w: Int) extends Bundle diff --git a/uncore/src/network.scala b/uncore/src/network.scala new file mode 100644 index 00000000..cdaa8118 --- /dev/null +++ b/uncore/src/network.scala @@ -0,0 +1,193 @@ +package uncore +import Chisel._ +import scala.collection.mutable.Stack + +class PairedDataIO[M <: Data, D <: Data]()(m: => M, d: => D) extends Bundle { + val meta = new FIFOIO()(m) + val data = new FIFOIO()(d) + override def clone = { new PairedDataIO()(m,d).asInstanceOf[this.type] } +} + +class PairedArbiterIO[M <: Data, D <: Data](n: Int)(m: => M, d: => D) extends Bundle { + val in = Vec(n) { new PairedDataIO()(m,d) }.flip + val out = new PairedDataIO()(m,d) + val meta_chosen = Bits(OUTPUT, log2Up(n)) + val data_chosen = Bits(OUTPUT, log2Up(n)) + override def clone = { new PairedArbiterIO(n)(m,d).asInstanceOf[this.type] } +} + +class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock: Option[M => Bool] = None)(meta: => M, data: => D) extends Component { + require(isPow2(count)) + val io = new PairedArbiterIO(n)(meta,data) + val locked = if(count > 1) Reg(resetVal = Bool(false)) else Bool(false) + val lockIdx = if(count > 1) Reg(resetVal = UFix(n-1)) else UFix(n-1) + val grant = List.fill(n)(Bool()) + val meta_chosen = Bits(width = log2Up(n)) + + val chosen_meta_has_data = needsLock.map(_(io.in(meta_chosen).meta.bits)).getOrElse(Bool(true)) + val valid_meta_has_data = io.in(meta_chosen).meta.valid && chosen_meta_has_data + val grant_chosen_meta = !(locked && chosen_meta_has_data) + (0 until n).map(i => io.in(i).meta.ready := grant(i) && grant_chosen_meta && io.out.meta.ready) + (0 until n).map(i => io.in(i).data.ready := Mux(locked, lockIdx === UFix(i), grant(i) && valid_meta_has_data) && io.out.data.ready) + io.out.meta.valid := io.in(meta_chosen).meta.valid && grant_chosen_meta + io.out.data.valid := Mux(locked, io.in(lockIdx).data.valid, io.in(meta_chosen).data.valid && valid_meta_has_data) + io.out.meta.bits := io.in(meta_chosen).meta.bits + io.out.data.bits := Mux(locked, io.in(lockIdx).data.bits, io.in(meta_chosen).data.bits) + io.meta_chosen := meta_chosen + io.data_chosen := Mux(locked, lockIdx, meta_chosen) + + if(count > 1){ + val cnt = Reg(resetVal = UFix(0, width = log2Up(count))) + val cnt_next = cnt + UFix(1) + when(io.out.data.fire()){ + cnt := cnt_next + when(cnt_next === UFix(0)) { + locked := Bool(false) + } + } + when(io.out.meta.fire()) { + when(needsLock.map(_(io.out.meta.bits)).getOrElse(Bool(true))) { + when(!locked) { + locked := Bool(true) + lockIdx := Vec(io.in.map{in => in.meta.fire()}){Bool()}.indexWhere{i: Bool => i} + } + } + } + } + val last_grant = Reg(resetVal = Bits(0, log2Up(n))) + val ctrl = ArbiterCtrl((0 until n).map(i => io.in(i).meta.valid && UFix(i) > last_grant) ++ io.in.map(_.meta.valid)) + (0 until n).map(i => grant(i) := ctrl(i) && UFix(i) > last_grant || ctrl(i + n)) + + var choose = Bits(n-1) + for (i <- n-2 to 0 by -1) + choose = Mux(io.in(i).meta.valid, Bits(i), choose) + for (i <- n-1 to 1 by -1) + choose = Mux(io.in(i).meta.valid && UFix(i) > last_grant, Bits(i), choose) + meta_chosen := choose + + when (io.out.meta.fire()) { last_grant := meta_chosen } +} + +class PairedCrossbar[M <: Data, D <: Data](count: Int, needsLock: Option[PhysicalNetworkIO[M] => Bool] = None)(meta: => M, data: => D)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { + val io = new Bundle { + val in = Vec(conf.nEndpoints){new PairedDataIO()(new PhysicalNetworkIO()(meta),new PhysicalNetworkIO()(data))}.flip + val out = Vec(conf.nEndpoints){new PairedDataIO()(new PhysicalNetworkIO()(meta),new PhysicalNetworkIO()(data))} + } + + val metaRdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) + val dataRdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) + val rdyVecs = metaRdyVecs zip dataRdyVecs + + io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { + val rrarb = new PairedLockingRRArbiter(conf.nEndpoints, count, needsLock)(io.in(0).meta.bits.clone, io.in(0).data.bits.clone) + rrarb.io.in zip io.in zip rdys._1 zip rdys._2 map { case (((arb, in), meta_rdy), data_rdy) => { + arb.meta.valid := in.meta.valid && (in.meta.bits.header.dst === UFix(i)) + arb.meta.bits := in.meta.bits + meta_rdy := arb.meta.ready && (in.meta.bits.header.dst === UFix(i)) + arb.data.valid := in.data.valid && (in.data.bits.header.dst === UFix(i)) + arb.data.bits := in.data.bits + data_rdy := arb.data.ready && (in.data.bits.header.dst === UFix(i)) + }} + out <> rrarb.io.out + }} + for(i <- 0 until conf.nEndpoints) { + io.in(i).meta.ready := rdyVecs.map(r => r._1(i)).reduceLeft(_||_) + io.in(i).data.ready := rdyVecs.map(r => r._2(i)).reduceLeft(_||_) + } +} + +case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) + +class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { + val src = UFix(width = conf.idBits) + val dst = UFix(width = conf.idBits) +} + +class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { + val header = (new PhysicalHeader) + val payload = data + override def clone = { new PhysicalNetworkIO()(data).asInstanceOf[this.type] } +} + +abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Component + +class BasicCrossbar[T <: Data](count: Int = 1)(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { + val io = new Bundle { + val in = Vec(conf.nEndpoints){(new FIFOIO){(new PhysicalNetworkIO){data}}}.flip + val out = Vec(conf.nEndpoints){(new FIFOIO){(new PhysicalNetworkIO){data}}} + } + + val rdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) + + io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { + val rrarb = (new LockingRRArbiter(conf.nEndpoints, count)){io.in(0).bits.clone} + (rrarb.io.in, io.in, rdys).zipped.map{ case (arb, in, rdy) => { + arb.valid := in.valid && (in.bits.header.dst === UFix(i)) + arb.bits := in.bits + rdy := arb.ready && (in.bits.header.dst === UFix(i)) + }} + out <> rrarb.io.out + }} + for(i <- 0 until conf.nEndpoints) { + io.in(i).ready := rdyVecs.map(r => r(i)).reduceLeft(_||_) + } +} + +case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nMasters: Int, nClients: Int) + +abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[CoherenceAgentRole])(implicit conf: LogicalNetworkConfiguration) extends Component { + override val io: Vec[TileLinkType] + val physicalNetworks: Seq[PhysicalNetwork] + require(endpoints.length == conf.nEndpoints) +} + +class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val src = UFix(width = conf.idBits) + val dst = UFix(width = conf.idBits) +} + +object FIFOedLogicalNetworkIOWrapper { + def apply[T <: Data](in: FIFOIO[T], src: UFix = UFix(0), dst: UFix = UFix(0))(implicit conf: LogicalNetworkConfiguration) = { + val shim = (new FIFOedLogicalNetworkIOWrapper(src, dst)){ in.bits.clone } + shim.io.in.valid := in.valid + shim.io.in.bits := in.bits + in.ready := shim.io.in.ready + shim.io.out + } +} +class FIFOedLogicalNetworkIOWrapper[T <: Data](src: UFix, dst: UFix)(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { + val io = new Bundle { + val in = (new FIFOIO){ data }.flip + val out = (new FIFOIO){(new LogicalNetworkIO){ data }} + } + io.out.valid := io.in.valid + io.out.bits.payload := io.in.bits + io.out.bits.header.dst := dst + io.out.bits.header.src := src + io.in.ready := io.out.ready +} + +object FIFOedLogicalNetworkIOUnwrapper { + def apply[T <: Data](in: FIFOIO[LogicalNetworkIO[T]])(implicit conf: LogicalNetworkConfiguration) = { + val shim = (new FIFOedLogicalNetworkIOUnwrapper){ in.bits.payload.clone } + shim.io.in.valid := in.valid + shim.io.in.bits := in.bits + in.ready := shim.io.in.ready + shim.io.out + } +} +class FIFOedLogicalNetworkIOUnwrapper[T <: Data]()(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { + val io = new Bundle { + val in = (new FIFOIO){(new LogicalNetworkIO){ data }}.flip + val out = (new FIFOIO){ data } + } + io.out.valid := io.in.valid + io.out.bits := io.in.bits.payload + io.in.ready := io.out.ready +} + +class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val header = new LogicalHeader + val payload = data + override def clone = { new LogicalNetworkIO()(data).asInstanceOf[this.type] } +} diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 754621ad..a92f739c 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -1,204 +1,8 @@ -package object uncore { -import Chisel._ -import Node._ -import scala.collection.mutable.Stack - -//TODO: Remove these Networking classes from the package object once Scala bug -//SI-3439 is resolved. - -implicit def toOption[A](a: A) = Option(a) - -class PairedDataIO[M <: Data, D <: Data]()(m: => M, d: => D) extends Bundle { - val meta = new FIFOIO()(m) - val data = new FIFOIO()(d) - override def clone = { new PairedDataIO()(m,d).asInstanceOf[this.type] } -} - -class PairedArbiterIO[M <: Data, D <: Data](n: Int)(m: => M, d: => D) extends Bundle { - val in = Vec(n) { new PairedDataIO()(m,d) }.flip - val out = new PairedDataIO()(m,d) - val meta_chosen = Bits(OUTPUT, log2Up(n)) - val data_chosen = Bits(OUTPUT, log2Up(n)) - override def clone = { new PairedArbiterIO(n)(m,d).asInstanceOf[this.type] } -} - -class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock: Option[M => Bool] = None)(meta: => M, data: => D) extends Component { - require(isPow2(count)) - val io = new PairedArbiterIO(n)(meta,data) - val locked = if(count > 1) Reg(resetVal = Bool(false)) else Bool(false) - val lockIdx = if(count > 1) Reg(resetVal = UFix(n-1)) else UFix(n-1) - val grant = List.fill(n)(Bool()) - val meta_chosen = Bits(width = log2Up(n)) - - val chosen_meta_has_data = needsLock.map(_(io.in(meta_chosen).meta.bits)).getOrElse(Bool(true)) - val valid_meta_has_data = io.in(meta_chosen).meta.valid && chosen_meta_has_data - val grant_chosen_meta = !(locked && chosen_meta_has_data) - (0 until n).map(i => io.in(i).meta.ready := grant(i) && grant_chosen_meta && io.out.meta.ready) - (0 until n).map(i => io.in(i).data.ready := Mux(locked, lockIdx === UFix(i), grant(i) && valid_meta_has_data) && io.out.data.ready) - io.out.meta.valid := io.in(meta_chosen).meta.valid && grant_chosen_meta - io.out.data.valid := Mux(locked, io.in(lockIdx).data.valid, io.in(meta_chosen).data.valid && valid_meta_has_data) - io.out.meta.bits := io.in(meta_chosen).meta.bits - io.out.data.bits := Mux(locked, io.in(lockIdx).data.bits, io.in(meta_chosen).data.bits) - io.meta_chosen := meta_chosen - io.data_chosen := Mux(locked, lockIdx, meta_chosen) - - if(count > 1){ - val cnt = Reg(resetVal = UFix(0, width = log2Up(count))) - val cnt_next = cnt + UFix(1) - when(io.out.data.fire()){ - cnt := cnt_next - when(cnt_next === UFix(0)) { - locked := Bool(false) - } - } - when(io.out.meta.fire()) { - when(needsLock.map(_(io.out.meta.bits)).getOrElse(Bool(true))) { - when(!locked) { - locked := Bool(true) - lockIdx := Vec(io.in.map{in => in.meta.fire()}){Bool()}.indexWhere{i: Bool => i} - } - } - } - } - val last_grant = Reg(resetVal = Bits(0, log2Up(n))) - val ctrl = ArbiterCtrl((0 until n).map(i => io.in(i).meta.valid && UFix(i) > last_grant) ++ io.in.map(_.meta.valid)) - (0 until n).map(i => grant(i) := ctrl(i) && UFix(i) > last_grant || ctrl(i + n)) - - var choose = Bits(n-1) - for (i <- n-2 to 0 by -1) - choose = Mux(io.in(i).meta.valid, Bits(i), choose) - for (i <- n-1 to 1 by -1) - choose = Mux(io.in(i).meta.valid && UFix(i) > last_grant, Bits(i), choose) - meta_chosen := choose - - when (io.out.meta.fire()) { last_grant := meta_chosen } -} - -class PairedCrossbar[M <: Data, D <: Data](count: Int, needsLock: Option[BasicCrossbarIO[M] => Bool] = None)(meta: => M, data: => D)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { - val io = new Bundle { - val in = Vec(conf.nEndpoints){new PairedDataIO()(new BasicCrossbarIO()(meta),new BasicCrossbarIO()(data))}.flip - val out = Vec(conf.nEndpoints){new PairedDataIO()(new BasicCrossbarIO()(meta),new BasicCrossbarIO()(data))} - } - - val metaRdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) - val dataRdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) - val rdyVecs = metaRdyVecs zip dataRdyVecs - - io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { - val rrarb = new PairedLockingRRArbiter(conf.nEndpoints, count, needsLock)(io.in(0).meta.bits.clone, io.in(0).data.bits.clone) - rrarb.io.in zip io.in zip rdys._1 zip rdys._2 map { case (((arb, in), meta_rdy), data_rdy) => { - arb.meta.valid := in.meta.valid && (in.meta.bits.header.dst === UFix(i)) - arb.meta.bits := in.meta.bits - meta_rdy := arb.meta.ready && (in.meta.bits.header.dst === UFix(i)) - arb.data.valid := in.data.valid && (in.data.bits.header.dst === UFix(i)) - arb.data.bits := in.data.bits - data_rdy := arb.data.ready && (in.data.bits.header.dst === UFix(i)) - }} - out <> rrarb.io.out - }} - for(i <- 0 until conf.nEndpoints) { - io.in(i).meta.ready := rdyVecs.map(r => r._1(i)).reduceLeft(_||_) - io.in(i).data.ready := rdyVecs.map(r => r._2(i)).reduceLeft(_||_) - } -} - -case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) - -class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { - val src = UFix(width = conf.idBits) - val dst = UFix(width = conf.idBits) -} - -abstract class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { - val header = (new PhysicalHeader) - val payload = data -} - -class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetworkIO()(data)(conf) { - override def clone = { new BasicCrossbarIO()(data).asInstanceOf[this.type] } -} - -abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Component - -class BasicCrossbar[T <: Data](count: Int = 1)(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { - val io = new Bundle { - val in = Vec(conf.nEndpoints){(new FIFOIO){(new BasicCrossbarIO){data}}}.flip - val out = Vec(conf.nEndpoints){(new FIFOIO){(new BasicCrossbarIO){data}}} - } - - val rdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) - - io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { - val rrarb = (new LockingRRArbiter(conf.nEndpoints, count)){io.in(0).bits.clone} - (rrarb.io.in, io.in, rdys).zipped.map{ case (arb, in, rdy) => { - arb.valid := in.valid && (in.bits.header.dst === UFix(i)) - arb.bits := in.bits - rdy := arb.ready && (in.bits.header.dst === UFix(i)) - }} - out <> rrarb.io.out - }} - for(i <- 0 until conf.nEndpoints) { - io.in(i).ready := rdyVecs.map(r => r(i)).reduceLeft(_||_) - } -} - -case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nMasters: Int, nClients: Int) - -abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[CoherenceAgentRole])(implicit conf: LogicalNetworkConfiguration) extends Component { - override val io: Vec[TileLinkType] - val physicalNetworks: Seq[PhysicalNetwork] - require(endpoints.length == conf.nEndpoints) -} - -class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val src = UFix(width = conf.idBits) - val dst = UFix(width = conf.idBits) -} - -object FIFOedLogicalNetworkIOWrapper { - def apply[T <: Data](in: FIFOIO[T], src: UFix = UFix(0), dst: UFix = UFix(0))(implicit conf: LogicalNetworkConfiguration) = { - val shim = (new FIFOedLogicalNetworkIOWrapper(src, dst)){ in.bits.clone } - shim.io.in.valid := in.valid - shim.io.in.bits := in.bits - in.ready := shim.io.in.ready - shim.io.out - } -} -class FIFOedLogicalNetworkIOWrapper[T <: Data](src: UFix, dst: UFix)(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { - val io = new Bundle { - val in = (new FIFOIO){ data }.flip - val out = (new FIFOIO){(new LogicalNetworkIO){ data }} - } - io.out.valid := io.in.valid - io.out.bits.payload := io.in.bits - io.out.bits.header.dst := dst - io.out.bits.header.src := src - io.in.ready := io.out.ready -} - -object FIFOedLogicalNetworkIOUnwrapper { - def apply[T <: Data](in: FIFOIO[LogicalNetworkIO[T]])(implicit conf: LogicalNetworkConfiguration) = { - val shim = (new FIFOedLogicalNetworkIOUnwrapper){ in.bits.payload.clone } - shim.io.in.valid := in.valid - shim.io.in.bits := in.bits - in.ready := shim.io.in.ready - shim.io.out - } -} -class FIFOedLogicalNetworkIOUnwrapper[T <: Data]()(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { - val io = new Bundle { - val in = (new FIFOIO){(new LogicalNetworkIO){ data }}.flip - val out = (new FIFOIO){ data } - } - io.out.valid := io.in.valid - io.out.bits := io.in.bits.payload - io.in.ready := io.out.ready -} - -class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val header = new LogicalHeader - val payload = data - override def clone = { new LogicalNetworkIO()(data).asInstanceOf[this.type] } -} - +package object uncore extends + uncore.constants.MemoryOpConstants with + uncore.constants.MemoryInterfaceConstants with + uncore.constants.CacheConstants with + uncore.constants.AddressConstants +{ + implicit def toOption[A](a: A) = Option(a) } diff --git a/uncore/src/slowio.scala b/uncore/src/slowio.scala index cf00ae6e..b56b1396 100644 --- a/uncore/src/slowio.scala +++ b/uncore/src/slowio.scala @@ -1,7 +1,5 @@ package uncore - import Chisel._ -import Constants._ class SlowIO[T <: Data](val divisor_max: Int)(data: => T) extends Component { diff --git a/uncore/src/temp.scala b/uncore/src/temp.scala deleted file mode 100644 index b2d3fdca..00000000 --- a/uncore/src/temp.scala +++ /dev/null @@ -1,13 +0,0 @@ -package uncore -import _root_.uncore.constants._ - -//TODO: When compiler bug SI-5604 is fixed in 2.10, remove object Constants and -// mixin Constants traits to package object uncore in package.scala and -// remove import Constants._'s from other .scala files -object Constants extends - MemoryOpConstants with - MemoryInterfaceConstants with - CacheConstants with - AddressConstants -{ -} diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index 2e595c34..3621e73b 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -1,7 +1,5 @@ package uncore - import Chisel._ -import Constants._ trait HasPhysicalAddress extends Bundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 9d0de31b..5f87f724 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -1,7 +1,5 @@ package uncore - import Chisel._ -import Constants._ class TrackerDependency(implicit conf: UncoreConfiguration) extends Bundle { val tracker_id = Bits(width = MASTER_XACT_ID_MAX_BITS) @@ -109,7 +107,6 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: Uncore val release_data_needs_write = Reg(resetVal = Bool(false)) val mem_cmd_sent = Reg(resetVal = Bool(false)) val cmd_to_write = co.getUncachedWriteAcquire(xact.addr, UFix(trackerId)) - val cmd_to_read = co.getUncachedReadAcquire(xact.addr, UFix(trackerId)) io.has_acquire_conflict := Bool(false) io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.meta.bits.payload.addr) && (state != s_idle) @@ -153,6 +150,7 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: Uncore when (release_data_needs_write) { doOuterReqWrite(io.master.acquire, io.client.release.data, + cmd_to_write, release_data_needs_write, mem_cmd_sent, init_client_id_) @@ -264,6 +262,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura when (release_data_needs_write) { doOuterReqWrite(io.master.acquire, io.client.release.data, + cmd_to_write, release_data_needs_write, r_w_mem_cmd_sent, release_data_client_id) @@ -273,17 +272,19 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura when (release_data_needs_write) { doOuterReqWrite(io.master.acquire, io.client.release.data, + cmd_to_write, release_data_needs_write, r_w_mem_cmd_sent, release_data_client_id) } . elsewhen(acquire_data_needs_write) { doOuterReqWrite(io.master.acquire, io.client.acquire.data, + cmd_to_write, acquire_data_needs_write, a_w_mem_cmd_sent, init_client_id_) } . elsewhen (x_needs_read) { - doOuterReqRead(io.master.acquire, x_needs_read) + doOuterReqRead(io.master.acquire, cmd_to_read, x_needs_read) } . otherwise { state := Mux(co.needsAckReply(xact.a_type, UFix(0)), s_ack, Mux(co.requiresAck(io.client.grant.bits.payload), s_busy, s_idle)) @@ -302,14 +303,12 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura } abstract trait OuterRequestGenerator { - val cmd_to_write: Acquire - val cmd_to_read: Acquire val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) - def doOuterReqWrite[T <: HasMemData](master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], client_data: FIFOIO[LogicalNetworkIO[T]], trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UFix) { + def doOuterReqWrite[T <: HasMemData](master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], client_data: FIFOIO[LogicalNetworkIO[T]], cmd: Acquire, trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UFix) { val do_write = client_data.valid && (client_data.bits.header.src === desired_client_data_src_id) - master_acq.meta.bits.payload := cmd_to_write + master_acq.meta.bits.payload := cmd master_acq.data.bits.payload := client_data.bits.payload when(master_acq.meta.fire()) { cmd_sent := Bool(true) @@ -329,9 +328,9 @@ abstract trait OuterRequestGenerator { } } - def doOuterReqRead(master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], trigger: Bool) { + def doOuterReqRead(master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], cmd: Acquire, trigger: Bool) { master_acq.meta.valid := Bool(true) - master_acq.meta.bits.payload := cmd_to_read + master_acq.meta.bits.payload := cmd when(master_acq.meta.ready) { trigger := Bool(false) } From bc2b45da129883748d197abab5fd50ec3a885a9a Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 2 Aug 2013 14:55:06 -0700 Subject: [PATCH 185/374] Fold uncore constants into TileLinkConfiguration, update coherence API --- uncore/src/coherence.scala | 229 +++++++++++++++++++++---------------- uncore/src/consts.scala | 61 ++++------ uncore/src/llc.scala | 12 +- uncore/src/memserdes.scala | 27 +++++ uncore/src/package.scala | 5 +- uncore/src/tilelink.scala | 122 ++++++++++---------- uncore/src/uncore.scala | 75 ++++++------ 7 files changed, 283 insertions(+), 248 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index b869436a..b8c3e4ad 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -6,6 +6,19 @@ abstract trait ClientCoherenceAgent extends CoherenceAgentRole abstract trait MasterCoherenceAgent extends CoherenceAgentRole abstract class CoherencePolicy { + def nClientStates: Int + def nMasterStates: Int + def nAcquireTypes: Int + def nProbeTypes: Int + def nReleaseTypes: Int + def nGrantTypes: Int + def clientStateBits = log2Up(nClientStates) + def masterStateBits = log2Up(nMasterStates) + def acquireTypeBits = log2Up(nAcquireTypes) + def probeTypeBits = log2Up(nProbeTypes) + def releaseTypeBits = log2Up(nReleaseTypes) + def grantTypeBits = log2Up(nGrantTypes) + def isHit (cmd: Bits, state: UFix): Bool def isValid (state: UFix): Bool @@ -22,11 +35,12 @@ abstract class CoherencePolicy { def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix + def getProbeType(a_type: UFix, global_state: UFix): UFix def getReleaseTypeOnCacheControl(cmd: Bits): Bits def getReleaseTypeOnVoluntaryWriteback(): Bits - - def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix): Release - def newRelease (incoming: Probe, state: UFix, id: UFix): Release + def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits + def getGrantType(a_type: UFix, count: UFix): Bits + def getGrantType(rel: Release, count: UFix): Bits def messageHasData (rel: SourcedMessage): Bool def messageUpdatesDataArray (reply: Grant): Bool @@ -35,9 +49,6 @@ abstract class CoherencePolicy { def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool def isVoluntary(rel: Release): Bool def isVoluntary(gnt: Grant): Bool - def getGrantType(a_type: UFix, count: UFix): Bits - def getGrantType(rel: Release, count: UFix): Bits - def getProbeType(a_type: UFix, global_state: UFix): UFix def needsOuterRead(a_type: UFix, global_state: UFix): Bool def needsOuterWrite(a_type: UFix, global_state: UFix): Bool def needsAckReply(a_type: UFix, global_state: UFix): Bool @@ -50,11 +61,11 @@ abstract class CoherencePolicy { } trait UncachedTransactions { - def getUncachedReadAcquire(addr: UFix, id: UFix): Acquire - def getUncachedWriteAcquire(addr: UFix, id: UFix): Acquire - def getUncachedReadWordAcquire(addr: UFix, id: UFix): Acquire - def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits): Acquire - def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix): Acquire + def getUncachedReadAcquireType: Bits + def getUncachedWriteAcquireType: Bits + def getUncachedReadWordAcquireType: Bits + def getUncachedWriteWordAcquireType: Bits + def getUncachedAtomicAcquireType: Bits def isUncachedReadTransaction(acq: Acquire): Bool } @@ -63,7 +74,7 @@ abstract class CoherencePolicyWithUncached extends CoherencePolicy with Uncached abstract class IncoherentPolicy extends CoherencePolicy { // UNIMPLEMENTED def newStateOnProbe(incoming: Probe, state: UFix): Bits = state - def newRelease (incoming: Probe, state: UFix, id: UFix): Release = Release( UFix(0), UFix(0), UFix(0), UFix(0)) + def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = Bits(0) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = Bool(false) def getGrantType(a_type: UFix, count: UFix): Bits = Bits(0) def getGrantType(rel: Release, count: UFix): Bits = Bits(0) @@ -79,10 +90,16 @@ abstract class IncoherentPolicy extends CoherencePolicy { } class ThreeStateIncoherence extends IncoherentPolicy { - val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } - val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(3){ UFix() } - val grantVoluntaryAck :: grantData :: grantAck :: Nil = Enum(3){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateAck :: Nil = Enum(2){ UFix() } + def nClientStates = 3 + def nMasterStates = 0 + def nAcquireTypes = 3 + def nProbeTypes = 0 + def nReleaseTypes = 2 + def nGrantTypes = 3 + val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(nClientStates){ UFix() } + val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(nAcquireTypes){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateAck :: Nil = Enum(nReleaseTypes){ UFix() } + val grantVoluntaryAck :: grantData :: grantAck :: Nil = Enum(nGrantTypes){ UFix() } val uncachedAcquireTypeList = List() val hasDataAcquireTypeList = List(acquireWriteback) val hasDataReleaseTypeList = List(acquireWriteback) @@ -109,7 +126,6 @@ class ThreeStateIncoherence extends IncoherentPolicy { )) } - def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck @@ -133,14 +149,21 @@ class ThreeStateIncoherence extends IncoherentPolicy { } class MICoherence extends CoherencePolicyWithUncached { + def nClientStates = 2 + def nMasterStates = 2 + def nAcquireTypes = 6 + def nProbeTypes = 2 + def nReleaseTypes = 5 + def nGrantTypes = 7 - val tileInvalid :: tileValid :: Nil = Enum(2){ UFix() } - val globalInvalid :: globalValid :: Nil = Enum(2){ UFix() } + val tileInvalid :: tileValid :: Nil = Enum(nClientStates){ UFix() } + val globalInvalid :: globalValid :: Nil = Enum(nMasterStates){ UFix() } + + val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UFix() } + val probeInvalidate :: probeCopy :: Nil = Enum(nProbeTypes){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UFix() } + val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UFix() } - val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(6){ UFix() } - val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(7){ UFix() } - val probeInvalidate :: probeCopy :: Nil = Enum(2){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(5){ UFix() } val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseCopyData) @@ -186,12 +209,11 @@ class MICoherence extends CoherencePolicyWithUncached { )) } - def getUncachedReadAcquire(addr: UFix, id: UFix) = Acquire(acquireReadUncached, addr, id) - def getUncachedWriteAcquire(addr: UFix, id: UFix) = Acquire(acquireWriteUncached, addr, id) - def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) - def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) - def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) - def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) + def getUncachedReadAcquireType = acquireReadUncached + def getUncachedWriteAcquireType = acquireWriteUncached + def getUncachedReadWordAcquireType = acquireReadWordUncached + def getUncachedWriteWordAcquireType = acquireWriteWordUncached + def getUncachedAtomicAcquireType = acquireAtomicUncached def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck @@ -200,8 +222,7 @@ class MICoherence extends CoherencePolicyWithUncached { def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = acquireReadExclusive def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - - def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { + def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeCopy -> releaseCopyData @@ -210,7 +231,7 @@ class MICoherence extends CoherencePolicyWithUncached { probeInvalidate -> releaseInvalidateAck, probeCopy -> releaseCopyAck )) - Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) + Mux(needsWriteback(state), with_data, without_data) } def messageHasData(msg: SourcedMessage) = msg match { @@ -270,14 +291,20 @@ class MICoherence extends CoherencePolicyWithUncached { } class MEICoherence extends CoherencePolicyWithUncached { + def nClientStates = 3 + def nMasterStates = 2 + def nAcquireTypes = 7 + def nProbeTypes = 3 + def nReleaseTypes = 7 + def nGrantTypes = 8 - val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(3){ UFix() } - val globalInvalid :: globalExclusiveClean :: Nil = Enum(2){ UFix() } + val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UFix() } + val globalInvalid :: globalExclusiveClean :: Nil = Enum(nMasterStates){ UFix() } - val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(7){ UFix() } - val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(8){ UFix() } - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(7){ UFix() } + val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UFix() } + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UFix() } + val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UFix() } val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -331,12 +358,11 @@ class MEICoherence extends CoherencePolicyWithUncached { )) } - def getUncachedReadAcquire(addr: UFix, id: UFix) = Acquire(acquireReadUncached, addr, id) - def getUncachedWriteAcquire(addr: UFix, id: UFix) = Acquire(acquireWriteUncached, addr, id) - def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) - def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) - def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) - def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) + def getUncachedReadAcquireType = acquireReadUncached + def getUncachedWriteAcquireType = acquireWriteUncached + def getUncachedReadWordAcquireType = acquireReadWordUncached + def getUncachedWriteWordAcquireType = acquireWriteWordUncached + def getUncachedAtomicAcquireType = acquireAtomicUncached def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck @@ -349,8 +375,7 @@ class MEICoherence extends CoherencePolicyWithUncached { } def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - - def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { + def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -361,7 +386,7 @@ class MEICoherence extends CoherencePolicyWithUncached { probeDowngrade -> releaseDowngradeAck, probeCopy -> releaseCopyAck )) - Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) + Mux(needsWriteback(state), with_data, without_data) } def messageHasData(msg: SourcedMessage) = msg match { @@ -424,14 +449,21 @@ class MEICoherence extends CoherencePolicyWithUncached { } class MSICoherence extends CoherencePolicyWithUncached { + def nClientStates = 3 + def nMasterStates = 3 + def nAcquireTypes = 7 + def nProbeTypes = 3 + def nReleaseTypes = 7 + def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveDirty :: Nil = Enum(3){ UFix() } - val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(3){ UFix() } + val tileInvalid :: tileShared :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UFix() } + val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(nMasterStates){ UFix() } + + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UFix() } + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UFix() } + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UFix() } - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(7){ UFix() } - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(9){ UFix() } - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(7){ UFix() } val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) @@ -490,12 +522,11 @@ class MSICoherence extends CoherencePolicyWithUncached { )) } - def getUncachedReadAcquire(addr: UFix, id: UFix) = Acquire(acquireReadUncached, addr, id) - def getUncachedWriteAcquire(addr: UFix, id: UFix) = Acquire(acquireWriteUncached, addr, id) - def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) - def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) - def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) - def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) + def getUncachedReadAcquireType = acquireReadUncached + def getUncachedWriteAcquireType = acquireWriteUncached + def getUncachedReadWordAcquireType = acquireReadWordUncached + def getUncachedWriteWordAcquireType = acquireWriteWordUncached + def getUncachedAtomicAcquireType = acquireAtomicUncached def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck @@ -508,8 +539,7 @@ class MSICoherence extends CoherencePolicyWithUncached { } def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - - def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { + def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -520,7 +550,7 @@ class MSICoherence extends CoherencePolicyWithUncached { probeDowngrade -> releaseDowngradeAck, probeCopy -> releaseCopyAck )) - Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) + Mux(needsWriteback(state), with_data, without_data) } def messageHasData(msg: SourcedMessage) = msg match { @@ -580,14 +610,20 @@ class MSICoherence extends CoherencePolicyWithUncached { } class MESICoherence extends CoherencePolicyWithUncached { + def nClientStates = 4 + def nMasterStates = 3 + def nAcquireTypes = 7 + def nProbeTypes = 3 + def nReleaseTypes = 7 + def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(4){ UFix() } - val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(3){ UFix() } + val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UFix() } + val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(nMasterStates){ UFix() } - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(7){ UFix() } - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(9){ UFix() } - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(7){ UFix() } + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UFix() } + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UFix() } + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UFix() } val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -647,12 +683,11 @@ class MESICoherence extends CoherencePolicyWithUncached { )) } - def getUncachedReadAcquire(addr: UFix, id: UFix) = Acquire(acquireReadUncached, addr, id) - def getUncachedWriteAcquire(addr: UFix, id: UFix) = Acquire(acquireWriteUncached, addr, id) - def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) - def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) - def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) - def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) + def getUncachedReadAcquireType = acquireReadUncached + def getUncachedWriteAcquireType = acquireWriteUncached + def getUncachedReadWordAcquireType = acquireReadWordUncached + def getUncachedWriteWordAcquireType = acquireWriteWordUncached + def getUncachedAtomicAcquireType = acquireAtomicUncached def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck @@ -665,8 +700,7 @@ class MESICoherence extends CoherencePolicyWithUncached { } def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - - def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { + def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -677,7 +711,7 @@ class MESICoherence extends CoherencePolicyWithUncached { probeDowngrade -> releaseDowngradeAck, probeCopy -> releaseCopyAck )) - Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) + Mux(needsWriteback(state), with_data, without_data) } def messageHasData(msg: SourcedMessage) = msg match { @@ -741,13 +775,19 @@ class MESICoherence extends CoherencePolicyWithUncached { } class MigratoryCoherence extends CoherencePolicyWithUncached { + def nClientStates = 7 + def nMasterStates = 0 + def nAcquireTypes = 8 + def nProbeTypes = 4 + def nReleaseTypes = 11 + def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: tileSharedByTwo :: tileMigratoryClean :: tileMigratoryDirty :: Nil = Enum(7){ UFix() } + val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: tileSharedByTwo :: tileMigratoryClean :: tileMigratoryDirty :: Nil = Enum(nClientStates){ UFix() } - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(8){ UFix() } - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(10){ UFix() } - val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(4){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(11){ UFix() } + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(nAcquireTypes){ UFix() } + val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(nProbeTypes){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(nReleaseTypes){ UFix() } + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(nGrantTypes){ UFix() } val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -820,12 +860,11 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { )) } - def getUncachedReadAcquire(addr: UFix, id: UFix) = Acquire(acquireReadUncached, addr, id) - def getUncachedWriteAcquire(addr: UFix, id: UFix) = Acquire(acquireWriteUncached, addr, id) - def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) - def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) - def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) - def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) + def getUncachedReadAcquireType = acquireReadUncached + def getUncachedWriteAcquireType = acquireWriteUncached + def getUncachedReadWordAcquireType = acquireReadWordUncached + def getUncachedWriteWordAcquireType = acquireWriteWordUncached + def getUncachedAtomicAcquireType = acquireAtomicUncached def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck @@ -838,22 +877,20 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - - def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { - //Assert( incoming.p_type === probeInvalidateOthers && needsWriteback(state), "Bad probe request type, should be impossible.") + def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> Mux(uFixListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), + probeInvalidate -> Mux(uFixListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), releaseInvalidateDataMigratory, releaseInvalidateData), - probeDowngrade -> Mux(state === tileMigratoryDirty, releaseDowngradeDataMigratory, releaseDowngradeData), - probeCopy -> releaseCopyData + probeDowngrade -> Mux(state === tileMigratoryDirty, releaseDowngradeDataMigratory, releaseDowngradeData), + probeCopy -> releaseCopyData )) val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( - probeInvalidate -> Mux(tileExclusiveClean === state, releaseInvalidateAckMigratory, releaseInvalidateAck), + probeInvalidate -> Mux(tileExclusiveClean === state, releaseInvalidateAckMigratory, releaseInvalidateAck), probeInvalidateOthers -> Mux(state === tileSharedByTwo, releaseInvalidateAckMigratory, releaseInvalidateAck), probeDowngrade -> Mux(state != tileInvalid, releaseDowngradeAckHasCopy, releaseDowngradeAck), probeCopy -> releaseCopyAck )) - Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) + Mux(needsWriteback(state), with_data, without_data) } def messageHasData(msg: SourcedMessage) = msg match { diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index 4e3c814e..15bfee4d 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -4,38 +4,6 @@ package constants import Chisel._ import scala.math.max -abstract trait CoherenceConfigConstants { - val ENABLE_SHARING: Boolean - val ENABLE_CLEAN_EXCLUSIVE: Boolean -} - -trait UncoreConstants { - val NGLOBAL_ACQ_XACTS = 8 - val NGLOBAL_REL_XACTS = 1 - val MASTER_XACT_ID_MAX_BITS = log2Up(NGLOBAL_ACQ_XACTS+NGLOBAL_REL_XACTS) - val CACHE_DATA_SIZE_IN_BYTES = 1 << 6 -} - -trait CacheConstants extends UncoreConstants { - val OFFSET_BITS = log2Up(CACHE_DATA_SIZE_IN_BYTES) -} - -trait TileLinkTypeConstants { - val ACQUIRE_TYPE_MAX_BITS = 2 - val GRANT_TYPE_MAX_BITS = 3 - val PROBE_TYPE_MAX_BITS = 2 - val RELEASE_TYPE_MAX_BITS = 3 -} - -trait TileLinkSizeConstants extends - TileLinkTypeConstants -{ - val CLIENT_XACT_ID_MAX_BITS = 10 - val ACQUIRE_WRITE_MASK_BITS = 6 - val ACQUIRE_SUBWORD_ADDR_BITS = 3 - val ACQUIRE_ATOMIC_OP_BITS = 4 -} - object MemoryOpConstants extends MemoryOpConstants trait MemoryOpConstants { val MT_X = Bits("b???", 3); @@ -74,15 +42,6 @@ trait MemoryOpConstants { def isWriteIntent(cmd: Bits) = isWrite(cmd) || cmd === M_PFW || cmd === M_XLR } -trait MemoryInterfaceConstants extends - UncoreConstants with - TileLinkSizeConstants -{ - val MEM_TAG_BITS = max(CLIENT_XACT_ID_MAX_BITS, MASTER_XACT_ID_MAX_BITS) - val MEM_DATA_BITS = 128 - val REFILL_CYCLES = CACHE_DATA_SIZE_IN_BYTES*8/MEM_DATA_BITS -} - object AddressConstants extends AddressConstants trait AddressConstants { val PADDR_BITS = 32 @@ -94,3 +53,23 @@ trait AddressConstants { val PERM_BITS = 6; } +trait CacheConstants { + val CACHE_DATA_SIZE_IN_BYTES = 1 << 6 + val OFFSET_BITS = log2Up(CACHE_DATA_SIZE_IN_BYTES) +} + +trait TileLinkSizeConstants { + val ACQUIRE_WRITE_MASK_BITS = 6 + val ACQUIRE_SUBWORD_ADDR_BITS = 3 + val ACQUIRE_ATOMIC_OP_BITS = 4 +} + +trait MemoryInterfaceConstants extends + CacheConstants with + AddressConstants +{ + val MEM_TAG_BITS = 5 + val MEM_DATA_BITS = 128 + val REFILL_CYCLES = CACHE_DATA_SIZE_IN_BYTES*8/MEM_DATA_BITS + val MEM_ADDR_BITS = PADDR_BITS - OFFSET_BITS +} diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index 6b0c77f2..cb6465c5 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -86,10 +86,10 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val cpu = (new FIFOIO) { new MemReqCmd }.flip val repl_way = UFix(INPUT, log2Up(ways)) val repl_dirty = Bool(INPUT) - val repl_tag = UFix(INPUT, PADDR_BITS - OFFSET_BITS - log2Up(sets)) + val repl_tag = UFix(INPUT, MEM_ADDR_BITS - log2Up(sets)) val data = (new FIFOIO) { new LLCDataReq(ways) } val tag = (new FIFOIO) { new Bundle { - val addr = UFix(width = PADDR_BITS - OFFSET_BITS) + val addr = UFix(width = MEM_ADDR_BITS) val way = UFix(width = log2Up(ways)) } } val mem = new ioMemPipe @@ -105,7 +105,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val refillCount = UFix(width = log2Up(REFILL_CYCLES)) val requested = Bool() val old_dirty = Bool() - val old_tag = UFix(width = PADDR_BITS - OFFSET_BITS - log2Up(sets)) + val old_tag = UFix(width = MEM_ADDR_BITS - log2Up(sets)) val wb_busy = Bool() override def clone = new MSHR().asInstanceOf[this.type] @@ -184,7 +184,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component class LLCWriteback(requestors: Int) extends Component { val io = new Bundle { - val req = Vec(requestors) { (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) }.flip } + val req = Vec(requestors) { (new FIFOIO) { UFix(width = MEM_ADDR_BITS) }.flip } val data = Vec(requestors) { (new FIFOIO) { new MemData }.flip } val mem = new ioMemPipe } @@ -235,7 +235,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Compo val io = new Bundle { val req = (new FIFOIO) { new LLCDataReq(ways) }.flip val req_data = (new FIFOIO) { new MemData }.flip - val writeback = (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) } + val writeback = (new FIFOIO) { UFix(width = MEM_ADDR_BITS) } val writeback_data = (new FIFOIO) { new MemData } val resp = (new FIFOIO) { new MemResp } val mem_resp = (new PipeIO) { new MemResp }.flip @@ -348,7 +348,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val mem = new ioMemPipe } - val tagWidth = PADDR_BITS - OFFSET_BITS - log2Up(sets) + val tagWidth = MEM_ADDR_BITS - log2Up(sets) val metaWidth = tagWidth + 2 // valid + dirty val memCmdArb = (new Arbiter(2)) { new MemReqCmd } diff --git a/uncore/src/memserdes.scala b/uncore/src/memserdes.scala index c7c8e773..b1272b28 100644 --- a/uncore/src/memserdes.scala +++ b/uncore/src/memserdes.scala @@ -2,6 +2,33 @@ package uncore import Chisel._ import scala.math._ +class MemData extends Bundle { + val data = Bits(width = MEM_DATA_BITS) +} + +class MemReqCmd extends Bundle { + val addr = UFix(width = MEM_ADDR_BITS) + val rw = Bool() + val tag = Bits(width = MEM_TAG_BITS) +} + +class MemResp extends Bundle { + val tag = Bits(width = MEM_TAG_BITS) + val data = Bits(width = MEM_DATA_BITS) +} + +class ioMem extends Bundle { + val req_cmd = (new FIFOIO) { new MemReqCmd() } + val req_data = (new FIFOIO) { new MemData() } + val resp = (new FIFOIO) { new MemResp() }.flip +} + +class ioMemPipe extends Bundle { + val req_cmd = (new FIFOIO) { new MemReqCmd() } + val req_data = (new FIFOIO) { new MemData() } + val resp = (new PipeIO) { new MemResp() }.flip +} + class ioMemSerialized(w: Int) extends Bundle { val req = (new FIFOIO) { Bits(width = w) } diff --git a/uncore/src/package.scala b/uncore/src/package.scala index a92f739c..4ac83d0b 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -1,8 +1,7 @@ package object uncore extends uncore.constants.MemoryOpConstants with - uncore.constants.MemoryInterfaceConstants with - uncore.constants.CacheConstants with - uncore.constants.AddressConstants + uncore.constants.TileLinkSizeConstants with + uncore.constants.MemoryInterfaceConstants { implicit def toOption[A](a: A) = Option(a) } diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index 3621e73b..6f7e6175 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -1,99 +1,86 @@ package uncore import Chisel._ -trait HasPhysicalAddress extends Bundle { +case class TileLinkConfiguration(co: CoherencePolicyWithUncached, ln: LogicalNetworkConfiguration, masterXactIdBits: Int, clientXactIdBits: Int, dataBits: Int) + +abstract trait TileLinkSubBundle extends Bundle { + implicit val conf: TileLinkConfiguration +} + +trait HasPhysicalAddress extends TileLinkSubBundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) } -trait HasClientTransactionId extends Bundle { - val client_xact_id = Bits(width = CLIENT_XACT_ID_MAX_BITS) +trait HasClientTransactionId extends TileLinkSubBundle { + val client_xact_id = Bits(width = conf.clientXactIdBits) } -trait HasMasterTransactionId extends Bundle { - val master_xact_id = Bits(width = MASTER_XACT_ID_MAX_BITS) +trait HasMasterTransactionId extends TileLinkSubBundle { + val master_xact_id = Bits(width = conf.masterXactIdBits) } -trait HasMemData extends Bundle { - val data = Bits(width = MEM_DATA_BITS) -} - -class MemData extends Bundle with HasMemData - -class MemReqCmd extends Bundle with HasPhysicalAddress { - val rw = Bool() - val tag = Bits(width = MEM_TAG_BITS) -} - -class MemResp extends Bundle with HasMemData { - val tag = Bits(width = MEM_TAG_BITS) -} - -class ioMem extends Bundle { - val req_cmd = (new FIFOIO) { new MemReqCmd() } - val req_data = (new FIFOIO) { new MemData() } - val resp = (new FIFOIO) { new MemResp() }.flip -} - -class ioMemPipe extends Bundle { - val req_cmd = (new FIFOIO) { new MemReqCmd() } - val req_data = (new FIFOIO) { new MemData() } - val resp = (new PipeIO) { new MemResp() }.flip +trait HasTileLinkData extends TileLinkSubBundle { + val data = Bits(width = conf.dataBits) } trait SourcedMessage extends Bundle trait ClientSourcedMessage extends SourcedMessage trait MasterSourcedMessage extends SourcedMessage -class Acquire extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId { - val a_type = Bits(width = ACQUIRE_TYPE_MAX_BITS) - val write_mask = Bits(width = ACQUIRE_WRITE_MASK_BITS) - val subword_addr = Bits(width = ACQUIRE_SUBWORD_ADDR_BITS) - val atomic_opcode = Bits(width = ACQUIRE_ATOMIC_OP_BITS) -} - object Acquire { - def apply(a_type: Bits, addr: UFix, client_xact_id: UFix) = { + def apply(a_type: Bits, addr: UFix, client_xact_id: UFix)(implicit conf: TileLinkConfiguration) = { val acq = new Acquire acq.a_type := a_type acq.addr := addr acq.client_xact_id := client_xact_id - acq.write_mask := Bits(0, width = ACQUIRE_WRITE_MASK_BITS) - acq.subword_addr := Bits(0, width = ACQUIRE_SUBWORD_ADDR_BITS) - acq.atomic_opcode := Bits(0, width = ACQUIRE_ATOMIC_OP_BITS) + acq.write_mask := Bits(0) + acq.subword_addr := Bits(0) + acq.atomic_opcode := Bits(0) acq } - def apply(a_type: Bits, addr: UFix, client_xact_id: UFix, write_mask: Bits) = { + def apply(a_type: Bits, addr: UFix, client_xact_id: UFix, write_mask: Bits)(implicit conf: TileLinkConfiguration) = { val acq = new Acquire acq.a_type := a_type acq.addr := addr acq.client_xact_id := client_xact_id acq.write_mask := write_mask - acq.subword_addr := Bits(0, width = ACQUIRE_SUBWORD_ADDR_BITS) - acq.atomic_opcode := Bits(0, width = ACQUIRE_ATOMIC_OP_BITS) + acq.subword_addr := Bits(0) + acq.atomic_opcode := Bits(0) acq } - def apply(a_type: Bits, addr: UFix, client_xact_id: UFix, subword_addr: UFix, atomic_opcode: UFix) = { + def apply(a_type: Bits, addr: UFix, client_xact_id: UFix, subword_addr: UFix, atomic_opcode: UFix)(implicit conf: TileLinkConfiguration) = { val acq = new Acquire acq.a_type := a_type acq.addr := addr acq.client_xact_id := client_xact_id acq.subword_addr := subword_addr acq.atomic_opcode := atomic_opcode - acq.write_mask := Bits(0, width = ACQUIRE_WRITE_MASK_BITS) + acq.write_mask := Bits(0) acq } } +class Acquire(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId { + val a_type = Bits(width = conf.co.acquireTypeBits) + val write_mask = Bits(width = ACQUIRE_WRITE_MASK_BITS) + val subword_addr = Bits(width = ACQUIRE_SUBWORD_ADDR_BITS) + val atomic_opcode = Bits(width = ACQUIRE_ATOMIC_OP_BITS) + override def clone = { (new Acquire).asInstanceOf[this.type] } +} -class AcquireData extends ClientSourcedMessage with HasMemData -class Probe extends MasterSourcedMessage with HasPhysicalAddress with HasMasterTransactionId { - val p_type = Bits(width = PROBE_TYPE_MAX_BITS) +class AcquireData(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasTileLinkData { + override def clone = { (new AcquireData).asInstanceOf[this.type] } +} + +class Probe(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage with HasPhysicalAddress with HasMasterTransactionId { + val p_type = Bits(width = conf.co.probeTypeBits) + override def clone = { (new Probe).asInstanceOf[this.type] } } object Release { - def apply(r_type: Bits, addr: UFix, client_xact_id: UFix, master_xact_id: UFix) = { + def apply(r_type: Bits, addr: UFix, client_xact_id: UFix, master_xact_id: UFix)(implicit conf: TileLinkConfiguration) = { val rel = new Release rel.r_type := r_type rel.addr := addr @@ -102,17 +89,24 @@ object Release rel } } -class Release extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasMasterTransactionId { - val r_type = Bits(width = RELEASE_TYPE_MAX_BITS) +class Release(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasMasterTransactionId { + val r_type = Bits(width = conf.co.releaseTypeBits) + override def clone = { (new Release).asInstanceOf[this.type] } } -class ReleaseData extends ClientSourcedMessage with HasMemData - -class Grant extends MasterSourcedMessage with HasMemData with HasClientTransactionId with HasMasterTransactionId { - val g_type = Bits(width = GRANT_TYPE_MAX_BITS) +class ReleaseData(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasTileLinkData { + override def clone = { (new ReleaseData).asInstanceOf[this.type] } +} + +class Grant(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage with HasTileLinkData with HasClientTransactionId with HasMasterTransactionId { + val g_type = Bits(width = conf.co.grantTypeBits) + override def clone = { (new Grant).asInstanceOf[this.type] } +} + +class GrantAck(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasMasterTransactionId { + override def clone = { (new GrantAck).asInstanceOf[this.type] } } -class GrantAck extends ClientSourcedMessage with HasMasterTransactionId trait DirectionalIO trait ClientSourcedIO extends DirectionalIO @@ -132,14 +126,15 @@ class MasterSourcedDataIO[M <: Data, D <: Data]()(meta: => M, data: => D) exten override def clone = { new MasterSourcedDataIO()(meta,data).asInstanceOf[this.type] } } -class UncachedTileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { +class UncachedTileLinkIO(implicit conf: TileLinkConfiguration) extends Bundle { + implicit val ln = conf.ln val acquire = new ClientSourcedDataIO()(new LogicalNetworkIO()(new Acquire), new LogicalNetworkIO()(new AcquireData)) val grant = new MasterSourcedFIFOIO()(new LogicalNetworkIO()(new Grant)) val grant_ack = new ClientSourcedFIFOIO()(new LogicalNetworkIO()(new GrantAck)) override def clone = { new UncachedTileLinkIO().asInstanceOf[this.type] } } -class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends UncachedTileLinkIO()(conf) { +class TileLinkIO(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIO()(conf) { val probe = new MasterSourcedFIFOIO()(new LogicalNetworkIO()(new Probe)) val release = new ClientSourcedDataIO()(new LogicalNetworkIO()(new Release), new LogicalNetworkIO()(new ReleaseData)) override def clone = { new TileLinkIO().asInstanceOf[this.type] } @@ -154,7 +149,8 @@ abstract class UncachedTileLinkIOArbiter(n: Int, co: CoherencePolicy)(implicit c } */ -class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Component { +class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int)(implicit conf: TileLinkConfiguration) extends Component { + implicit val (ln, co) = (conf.ln, conf.co) def acquireClientXactId(in: Acquire, id: Int) = Cat(in.client_xact_id, UFix(id, log2Up(n))) def grantClientXactId(in: Grant) = in.client_xact_id >> UFix(log2Up(n)) def arbIdx(in: Grant) = in.client_xact_id(log2Up(n)-1,0).toUFix @@ -190,7 +186,8 @@ class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int, co: CoherencePolicy) } } -class UncachedTileLinkIOArbiterThatPassesId(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Component { +class UncachedTileLinkIOArbiterThatPassesId(n: Int)(implicit conf: TileLinkConfiguration) extends Component { + implicit val (ln, co) = (conf.ln, conf.co) def acquireClientXactId(in: Acquire, id: Int) = in.client_xact_id def grantClientXactId(in: Grant) = in.client_xact_id def arbIdx(in: Grant): UFix = in.client_xact_id @@ -226,7 +223,8 @@ class UncachedTileLinkIOArbiterThatPassesId(n: Int, co: CoherencePolicy)(implici } } -class UncachedTileLinkIOArbiterThatUsesNewId(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Component { +class UncachedTileLinkIOArbiterThatUsesNewId(n: Int)(implicit conf: TileLinkConfiguration) extends Component { + implicit val (ln, co) = (conf.ln, conf.co) def acquireClientXactId(in: Acquire, id: Int) = UFix(id, log2Up(n)) def grantClientXactId(in: Grant) = UFix(0) // DNC def arbIdx(in: Grant) = in.client_xact_id diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 5f87f724..60e100e0 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -1,30 +1,26 @@ package uncore import Chisel._ -class TrackerDependency(implicit conf: UncoreConfiguration) extends Bundle { - val tracker_id = Bits(width = MASTER_XACT_ID_MAX_BITS) - val data_src_id = Bits(width = conf.ln.idBits) - override def clone = { new TrackerDependency().asInstanceOf[this.type] } -} - -case class UncoreConfiguration(co: CoherencePolicyWithUncached, ln: LogicalNetworkConfiguration) - -abstract class CoherenceAgent(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { +abstract class CoherenceAgent(implicit conf: TileLinkConfiguration) extends Component with MasterCoherenceAgent { val io = new Bundle { val client = (new TileLinkIO).flip val master = new UncachedTileLinkIO - val incoherent = Vec(conf.nClients) { Bool() }.asInput + val incoherent = Vec(conf.ln.nClients) { Bool() }.asInput } } -class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends CoherenceAgent()(conf.ln) +case class L2CoherenceAgentConfiguration(tl: TileLinkConfiguration, nReleaseTransactions: Int, nAcquireTransactions: Int) + +class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends CoherenceAgent()(conf.tl) { - implicit val lnConf = conf.ln - val co = conf.co - //require(conf.ln.nClients < NGLOBAL_REL_XACTS) //TODO: handle in config - val trackerList = (0 until NGLOBAL_REL_XACTS).map(new VoluntaryReleaseTracker(_, bankId)) ++ - (NGLOBAL_REL_XACTS until NGLOBAL_REL_XACTS + NGLOBAL_ACQ_XACTS).map(new AcquireTracker(_, bankId)) + implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) + + // Create SHRs for outstanding transactions + val nTrackers = conf.nReleaseTransactions + conf.nAcquireTransactions + val trackerList = (0 until conf.nReleaseTransactions).map(new VoluntaryReleaseTracker(_, bankId)) ++ + (conf.nReleaseTransactions until nTrackers).map(new AcquireTracker(_, bankId)) + // Propagate incoherence flags trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) // Handle acquire transaction initiation @@ -81,32 +77,31 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends ack.ready := Bool(true) // Create an arbiter for the one memory port - val outer_arb = new UncachedTileLinkIOArbiterThatPassesId(trackerList.size, conf.co) + val outer_arb = new UncachedTileLinkIOArbiterThatPassesId(trackerList.size) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.master } io.master <> outer_arb.io.out } -abstract class XactTracker()(implicit conf: UncoreConfiguration) extends Component with OuterRequestGenerator { - val co = conf.co - implicit val ln = conf.ln +abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) extends Component with OuterRequestGenerator { + implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val io = new Bundle { val client = (new TileLinkIO).flip val master = new UncachedTileLinkIO - val tile_incoherent = Bits(INPUT, conf.ln.nClients) + val tile_incoherent = Bits(INPUT, ln.nClients) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) } } -class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfiguration) extends XactTracker()(conf) { +class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(4){ UFix() } val state = Reg(resetVal = s_idle) val xact = Reg{ new Release } - val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) + val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(ln.nClients))) val release_data_needs_write = Reg(resetVal = Bool(false)) val mem_cmd_sent = Reg(resetVal = Bool(false)) - val cmd_to_write = co.getUncachedWriteAcquire(xact.addr, UFix(trackerId)) + val cmd_to_write = Acquire(co.getUncachedWriteAcquireType, xact.addr, UFix(trackerId)) io.has_acquire_conflict := Bool(false) io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.meta.bits.payload.addr) && (state != s_idle) @@ -163,38 +158,38 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: Uncore } } -class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfiguration) extends XactTracker()(conf) { +class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) val xact = Reg{ new Acquire } - val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) - val release_data_client_id = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) + val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(ln.nClients))) + val release_data_client_id = Reg(resetVal = UFix(0, width = log2Up(ln.nClients))) //TODO: Will need id reg for merged release xacts - val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) + val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(ln.nClients))) val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt_) - val release_count = if (conf.ln.nClients == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) - val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nClients)) + val release_count = if (ln.nClients == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(ln.nClients))) + val probe_flags = Reg(resetVal = Bits(0, width = ln.nClients)) val curr_p_id = PriorityEncoder(probe_flags) val x_needs_read = Reg(resetVal = Bool(false)) val acquire_data_needs_write = Reg(resetVal = Bool(false)) val release_data_needs_write = Reg(resetVal = Bool(false)) - val cmd_to_write = co.getUncachedWriteAcquire(xact.addr, UFix(trackerId)) - val cmd_to_read = co.getUncachedReadAcquire(xact.addr, UFix(trackerId)) + val cmd_to_write = Acquire(co.getUncachedWriteAcquireType, xact.addr, UFix(trackerId)) + val cmd_to_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UFix(trackerId)) val a_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val r_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val probe_initial_flags = Bits(width = conf.ln.nClients) + val probe_initial_flags = Bits(width = ln.nClients) probe_initial_flags := Bits(0) - if (conf.ln.nClients > 1) { + if (ln.nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence val probe_self = Bool(true) //co.needsSelfProbe(io.client.acquire.bits.payload) - val myflag = Mux(probe_self, Bits(0), UFixToOH(io.client.acquire.meta.bits.header.src(log2Up(conf.ln.nClients)-1,0))) + val myflag = Mux(probe_self, Bits(0), UFixToOH(io.client.acquire.meta.bits.header.src(log2Up(ln.nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, io.client.acquire.meta.bits.payload.addr) && (state != s_idle) io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.meta.bits.payload.addr) && (state != s_idle) io.master.acquire.meta.valid := Bool(false) - io.master.acquire.meta.bits.payload := co.getUncachedReadAcquire(xact.addr, UFix(trackerId)) + io.master.acquire.meta.bits.payload := Acquire(co.getUncachedReadAcquireType, xact.addr, UFix(trackerId)) //TODO io.master.acquire.bits.header.dst io.master.acquire.meta.bits.header.src := UFix(bankId) io.master.acquire.data.valid := Bool(false) @@ -227,14 +222,14 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura when( io.client.acquire.meta.valid ) { xact := io.client.acquire.meta.bits.payload init_client_id_ := io.client.acquire.meta.bits.header.src - init_sharer_cnt_ := UFix(conf.ln.nClients) // TODO: Broadcast only + init_sharer_cnt_ := UFix(ln.nClients) // TODO: Broadcast only acquire_data_needs_write := co.messageHasData(io.client.acquire.meta.bits.payload) x_needs_read := co.needsOuterRead(io.client.acquire.meta.bits.payload.a_type, UFix(0)) probe_flags := probe_initial_flags mem_cnt := UFix(0) r_w_mem_cmd_sent := Bool(false) a_w_mem_cmd_sent := Bool(false) - if(conf.ln.nClients > 1) { + if(ln.nClients > 1) { release_count := PopCount(probe_initial_flags) state := Mux(probe_initial_flags.orR, s_probe, s_mem) } else state := s_mem @@ -250,7 +245,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura } io.client.release.meta.ready := Bool(true) when(io.client.release.meta.valid) { - if(conf.ln.nClients > 1) release_count := release_count - UFix(1) + if(ln.nClients > 1) release_count := release_count - UFix(1) when(release_count === UFix(1)) { state := s_mem } @@ -306,7 +301,7 @@ abstract trait OuterRequestGenerator { val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) - def doOuterReqWrite[T <: HasMemData](master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], client_data: FIFOIO[LogicalNetworkIO[T]], cmd: Acquire, trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UFix) { + def doOuterReqWrite[T <: HasTileLinkData](master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], client_data: FIFOIO[LogicalNetworkIO[T]], cmd: Acquire, trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UFix) { val do_write = client_data.valid && (client_data.bits.header.src === desired_client_data_src_id) master_acq.meta.bits.payload := cmd master_acq.data.bits.payload := client_data.bits.payload From 5c7a1f5cd6b4331807e5899f30e1a9348cc8d428 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 12 Aug 2013 10:36:44 -0700 Subject: [PATCH 186/374] initial attempt at upgrade --- uncore/src/coherence.scala | 426 ++++++++++++++++++------------------- uncore/src/llc.scala | 234 ++++++++++---------- uncore/src/memserdes.scala | 129 +++++------ uncore/src/network.scala | 92 ++++---- uncore/src/slowio.scala | 61 +++--- uncore/src/tilelink.scala | 69 +++--- uncore/src/uncore.scala | 134 ++++++------ 7 files changed, 574 insertions(+), 571 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index b8c3e4ad..7109be94 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -12,35 +12,35 @@ abstract class CoherencePolicy { def nProbeTypes: Int def nReleaseTypes: Int def nGrantTypes: Int - def clientStateBits = log2Up(nClientStates) - def masterStateBits = log2Up(nMasterStates) - def acquireTypeBits = log2Up(nAcquireTypes) - def probeTypeBits = log2Up(nProbeTypes) - def releaseTypeBits = log2Up(nReleaseTypes) - def grantTypeBits = log2Up(nGrantTypes) + def clientStateWidth = log2Up(nClientStates) + def masterStateWidth = log2Up(nMasterStates) + def acquireTypeWidth = log2Up(nAcquireTypes) + def probeTypeWidth = log2Up(nProbeTypes) + def releaseTypeWidth = log2Up(nReleaseTypes) + def grantTypeWidth = log2Up(nGrantTypes) - def isHit (cmd: Bits, state: UFix): Bool - def isValid (state: UFix): Bool + def isHit (cmd: UInt, state: UInt): Bool + def isValid (state: UInt): Bool - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool - def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool - def needsWriteback (state: UFix): Bool + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool + def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool + def needsWriteback (state: UInt): Bool - def newStateOnHit(cmd: Bits, state: UFix): UFix - def newStateOnCacheControl(cmd: Bits): UFix - def newStateOnWriteback(): UFix - def newStateOnFlush(): UFix - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix - def newStateOnProbe(incoming: Probe, state: UFix): Bits + def newStateOnHit(cmd: UInt, state: UInt): UInt + def newStateOnCacheControl(cmd: UInt): UInt + def newStateOnWriteback(): UInt + def newStateOnFlush(): UInt + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt + def newStateOnProbe(incoming: Probe, state: UInt): Bits - def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix - def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix - def getProbeType(a_type: UFix, global_state: UFix): UFix - def getReleaseTypeOnCacheControl(cmd: Bits): Bits + def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt + def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt + def getProbeType(a_type: UInt, global_state: UInt): UInt + def getReleaseTypeOnCacheControl(cmd: UInt): Bits def getReleaseTypeOnVoluntaryWriteback(): Bits - def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits - def getGrantType(a_type: UFix, count: UFix): Bits - def getGrantType(rel: Release, count: UFix): Bits + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits + def getGrantType(a_type: UInt, count: UInt): Bits + def getGrantType(rel: Release, count: UInt): Bits def messageHasData (rel: SourcedMessage): Bool def messageUpdatesDataArray (reply: Grant): Bool @@ -49,15 +49,15 @@ abstract class CoherencePolicy { def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool def isVoluntary(rel: Release): Bool def isVoluntary(gnt: Grant): Bool - def needsOuterRead(a_type: UFix, global_state: UFix): Bool - def needsOuterWrite(a_type: UFix, global_state: UFix): Bool - def needsAckReply(a_type: UFix, global_state: UFix): Bool + def needsOuterRead(a_type: UInt, global_state: UInt): Bool + def needsOuterWrite(a_type: UInt, global_state: UInt): Bool + def needsAckReply(a_type: UInt, global_state: UInt): Bool def needsSelfProbe(acq: Acquire): Bool def requiresAck(grant: Grant): Bool def requiresAck(release: Release): Bool - def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool + def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool - def uFixListContains(list: List[UFix], elem: UFix): Bool = list.map(elem === _).reduceLeft(_||_) + def uSIntListContains(list: List[UInt], elem: UInt): Bool = list.map(elem === _).reduceLeft(_||_) } trait UncachedTransactions { @@ -73,19 +73,19 @@ abstract class CoherencePolicyWithUncached extends CoherencePolicy with Uncached abstract class IncoherentPolicy extends CoherencePolicy { // UNIMPLEMENTED - def newStateOnProbe(incoming: Probe, state: UFix): Bits = state - def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = Bits(0) + def newStateOnProbe(incoming: Probe, state: UInt): Bits = state + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = Bits(0) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = Bool(false) - def getGrantType(a_type: UFix, count: UFix): Bits = Bits(0) - def getGrantType(rel: Release, count: UFix): Bits = Bits(0) - def getProbeType(a_type: UFix, global_state: UFix): UFix = UFix(0) - def needsOuterRead(a_type: UFix, global_state: UFix): Bool = Bool(false) - def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = Bool(false) - def needsAckReply(a_type: UFix, global_state: UFix): Bool = Bool(false) + def getGrantType(a_type: UInt, count: UInt): Bits = Bits(0) + def getGrantType(rel: Release, count: UInt): Bits = Bits(0) + def getProbeType(a_type: UInt, global_state: UInt): UInt = UInt(0) + def needsOuterRead(a_type: UInt, global_state: UInt): Bool = Bool(false) + def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = Bool(false) + def needsAckReply(a_type: UInt, global_state: UInt): Bool = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) - def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = Bool(false) + def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = Bool(false) } @@ -96,27 +96,27 @@ class ThreeStateIncoherence extends IncoherentPolicy { def nProbeTypes = 0 def nReleaseTypes = 2 def nGrantTypes = 3 - val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(nClientStates){ UFix() } - val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(nAcquireTypes){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateAck :: Nil = Enum(nReleaseTypes){ UFix() } - val grantVoluntaryAck :: grantData :: grantAck :: Nil = Enum(nGrantTypes){ UFix() } + val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(nClientStates){ UInt() } + val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(nAcquireTypes){ UInt() } + val releaseVoluntaryInvalidateData :: releaseInvalidateAck :: Nil = Enum(nReleaseTypes){ UInt() } + val grantVoluntaryAck :: grantData :: grantAck :: Nil = Enum(nGrantTypes){ UInt() } val uncachedAcquireTypeList = List() val hasDataAcquireTypeList = List(acquireWriteback) val hasDataReleaseTypeList = List(acquireWriteback) val hasDataGrantTypeList = List(grantData) - def isHit ( cmd: Bits, state: UFix): Bool = (state === tileClean || state === tileDirty) - def isValid (state: UFix): Bool = state != tileInvalid + def isHit ( cmd: UInt, state: UInt): Bool = (state === tileClean || state === tileDirty) + def isValid (state: UInt): Bool = state != tileInvalid - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire) = Bool(false) - def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = state === tileDirty - def needsWriteback (state: UFix): Bool = state === tileDirty + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire) = Bool(false) + def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = state === tileDirty + def needsWriteback (state: UInt): Bool = state === tileDirty - def newState(cmd: Bits, state: UFix): UFix = { + def newState(cmd: UInt, state: UInt): UInt = { Mux(isWrite(cmd), tileDirty, Mux(isRead(cmd), Mux(state === tileDirty, tileDirty, tileClean), state)) } - def newStateOnHit(cmd: Bits, state: UFix): UFix = newState(cmd, state) - def newStateOnCacheControl(cmd: Bits) = tileInvalid //TODO + def newStateOnHit(cmd: UInt, state: UInt): UInt = newState(cmd, state) + def newStateOnCacheControl(cmd: UInt) = tileInvalid //TODO def newStateOnWriteback() = tileInvalid def newStateOnFlush() = tileInvalid def newStateOnGrant(incoming: Grant, outstanding: Acquire) = { @@ -129,23 +129,23 @@ class ThreeStateIncoherence extends IncoherentPolicy { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = { Mux(isWriteIntent(cmd), acquireReadDirty, acquireReadClean) } - def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { + def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadDirty, outstanding.a_type) } - def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = releaseVoluntaryInvalidateData def messageHasData( msg: SourcedMessage ) = msg match { - case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) + case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) case rel: Release => Bool(false) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant) = (reply.g_type === grantData) - def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) } class MICoherence extends CoherencePolicyWithUncached { @@ -156,35 +156,35 @@ class MICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 5 def nGrantTypes = 7 - val tileInvalid :: tileValid :: Nil = Enum(nClientStates){ UFix() } - val globalInvalid :: globalValid :: Nil = Enum(nMasterStates){ UFix() } + val tileInvalid :: tileValid :: Nil = Enum(nClientStates){ UInt() } + val globalInvalid :: globalValid :: Nil = Enum(nMasterStates){ UInt() } - val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UFix() } - val probeInvalidate :: probeCopy :: Nil = Enum(nProbeTypes){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UFix() } - val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UFix() } + val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UInt() } + val probeInvalidate :: probeCopy :: Nil = Enum(nProbeTypes){ UInt() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UInt() } + val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UInt() } val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseCopyData) val hasDataGrantTypeList = List(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) - def isHit (cmd: Bits, state: UFix): Bool = state != tileInvalid - def isValid (state: UFix): Bool = state != tileInvalid + def isHit (cmd: UInt, state: UInt): Bool = state != tileInvalid + def isValid (state: UInt): Bool = state != tileInvalid - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = (outstanding.a_type != acquireReadExclusive) - def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = (outstanding.a_type != acquireReadExclusive) + def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { MuxLookup(cmd, (state === tileValid), Array( M_INV -> (state === tileValid), M_CLN -> (state === tileValid) )) } - def needsWriteback (state: UFix): Bool = { + def needsWriteback (state: UInt): Bool = { needsTransactionOnCacheControl(M_INV, state) } - def newStateOnHit(cmd: Bits, state: UFix): UFix = state - def newStateOnCacheControl(cmd: Bits) = { + def newStateOnHit(cmd: UInt, state: UInt): UInt = state + def newStateOnCacheControl(cmd: UInt) = { MuxLookup(cmd, tileInvalid, Array( M_INV -> tileInvalid, M_CLN -> tileValid @@ -192,7 +192,7 @@ class MICoherence extends CoherencePolicyWithUncached { } def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix = { + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt = { MuxLookup(incoming.g_type, tileInvalid, Array( grantReadExclusive -> tileValid, grantReadUncached -> tileInvalid, @@ -202,7 +202,7 @@ class MICoherence extends CoherencePolicyWithUncached { grantAtomicUncached -> tileInvalid )) } - def newStateOnProbe(incoming: Probe, state: UFix): Bits = { + def newStateOnProbe(incoming: Probe, state: UInt): Bits = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeCopy -> state @@ -218,11 +218,11 @@ class MICoherence extends CoherencePolicyWithUncached { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = acquireReadExclusive - def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = acquireReadExclusive - def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = acquireReadExclusive + def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = acquireReadExclusive + def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = { + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeCopy -> releaseCopyData @@ -235,19 +235,19 @@ class MICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uFixListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uSIntListContains(hasDataReleaseTypeList, rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getGrantType(a_type: UFix, count: UFix): Bits = { + def getGrantType(a_type: UInt, count: UInt): Bits = { MuxLookup(a_type, grantReadUncached, Array( acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, @@ -258,13 +258,13 @@ class MICoherence extends CoherencePolicyWithUncached { )) } - def getGrantType(rel: Release, count: UFix): Bits = { + def getGrantType(rel: Release, count: UInt): Bits = { MuxLookup(rel.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) } - def getProbeType(a_type: UFix, global_state: UFix): UFix = { + def getProbeType(a_type: UInt, global_state: UInt): UInt = { MuxLookup(a_type, probeCopy, Array( acquireReadExclusive -> probeInvalidate, acquireReadUncached -> probeCopy, @@ -275,19 +275,19 @@ class MICoherence extends CoherencePolicyWithUncached { )) } - def needsOuterRead(a_type: UFix, global_state: UFix): Bool = { + def needsOuterRead(a_type: UInt, global_state: UInt): Bool = { (a_type != acquireWriteUncached) } - def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = { + def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def needsAckReply(a_type: UFix, global_state: UFix): Bool = { + def needsAckReply(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck def requiresAck(release: Release) = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) - def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) + def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } class MEICoherence extends CoherencePolicyWithUncached { @@ -298,40 +298,40 @@ class MEICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 7 def nGrantTypes = 8 - val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UFix() } - val globalInvalid :: globalExclusiveClean :: Nil = Enum(nMasterStates){ UFix() } + val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UInt() } + val globalInvalid :: globalExclusiveClean :: Nil = Enum(nMasterStates){ UInt() } - val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UFix() } - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UFix() } - val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UFix() } + val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UInt() } + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UInt() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UInt() } + val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UInt() } val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) val hasDataGrantTypeList = List(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) - def isHit (cmd: Bits, state: UFix): Bool = state != tileInvalid - def isValid (state: UFix): Bool = state != tileInvalid + def isHit (cmd: UInt, state: UInt): Bool = state != tileInvalid + def isValid (state: UInt): Bool = state != tileInvalid - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && messageIsUncached(outstanding)) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusiveDirty)) } - def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { + def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( M_INV -> (state === tileExclusiveDirty), M_CLN -> (state === tileExclusiveDirty) )) } - def needsWriteback (state: UFix): Bool = { + def needsWriteback (state: UInt): Bool = { needsTransactionOnCacheControl(M_INV, state) } - def newStateOnHit(cmd: Bits, state: UFix): UFix = { + def newStateOnHit(cmd: UInt, state: UInt): UInt = { Mux(isWrite(cmd), tileExclusiveDirty, state) } - def newStateOnCacheControl(cmd: Bits) = { + def newStateOnCacheControl(cmd: UInt) = { MuxLookup(cmd, tileInvalid, Array( M_INV -> tileInvalid, M_CLN -> tileExclusiveClean @@ -339,7 +339,7 @@ class MEICoherence extends CoherencePolicyWithUncached { } def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix = { + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt = { MuxLookup(incoming.g_type, tileInvalid, Array( grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusiveDirty, tileExclusiveDirty, tileExclusiveClean), grantReadExclusiveAck -> tileExclusiveDirty, @@ -350,7 +350,7 @@ class MEICoherence extends CoherencePolicyWithUncached { grantAtomicUncached -> tileInvalid )) } - def newStateOnProbe(incoming: Probe, state: UFix): Bits = { + def newStateOnProbe(incoming: Probe, state: UInt): Bits = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeDowngrade -> tileExclusiveClean, @@ -367,15 +367,15 @@ class MEICoherence extends CoherencePolicyWithUncached { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, acquireReadExclusiveClean) } - def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { + def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, outstanding.a_type) } - def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = { + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -390,19 +390,19 @@ class MEICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uFixListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uSIntListContains(hasDataReleaseTypeList, rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getGrantType(a_type: UFix, count: UFix): Bits = { + def getGrantType(a_type: UInt, count: UInt): Bits = { MuxLookup(a_type, grantReadUncached, Array( acquireReadExclusiveClean -> grantReadExclusive, acquireReadExclusiveDirty -> grantReadExclusive, @@ -413,14 +413,14 @@ class MEICoherence extends CoherencePolicyWithUncached { acquireAtomicUncached -> grantAtomicUncached )) } - def getGrantType(rel: Release, count: UFix): Bits = { + def getGrantType(rel: Release, count: UInt): Bits = { MuxLookup(rel.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) } - def getProbeType(a_type: UFix, global_state: UFix): UFix = { + def getProbeType(a_type: UInt, global_state: UInt): UInt = { MuxLookup(a_type, probeCopy, Array( acquireReadExclusiveClean -> probeInvalidate, acquireReadExclusiveDirty -> probeInvalidate, @@ -432,20 +432,20 @@ class MEICoherence extends CoherencePolicyWithUncached { )) } - def needsOuterRead(a_type: UFix, global_state: UFix): Bool = { + def needsOuterRead(a_type: UInt, global_state: UInt): Bool = { (a_type != acquireWriteUncached) } - def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = { + def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def needsAckReply(a_type: UFix, global_state: UFix): Bool = { + def needsAckReply(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck def requiresAck(release: Release) = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) - def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) + def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } class MSICoherence extends CoherencePolicyWithUncached { @@ -456,45 +456,45 @@ class MSICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 7 def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UFix() } - val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(nMasterStates){ UFix() } + val tileInvalid :: tileShared :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UInt() } + val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(nMasterStates){ UInt() } - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UFix() } - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UFix() } - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UFix() } + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UInt() } + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UInt() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UInt() } + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UInt() } val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) - def isHit (cmd: Bits, state: UFix): Bool = { + def isHit (cmd: UInt, state: UInt): Bool = { Mux(isWriteIntent(cmd), (state === tileExclusiveDirty), (state === tileShared || state === tileExclusiveDirty)) } - def isValid (state: UFix): Bool = { + def isValid (state: UInt): Bool = { state != tileInvalid } - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && messageIsUncached(outstanding)) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) } - def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { + def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( M_INV -> (state === tileExclusiveDirty), M_CLN -> (state === tileExclusiveDirty) )) } - def needsWriteback (state: UFix): Bool = { + def needsWriteback (state: UInt): Bool = { needsTransactionOnCacheControl(M_INV, state) } - def newStateOnHit(cmd: Bits, state: UFix): UFix = { + def newStateOnHit(cmd: UInt, state: UInt): UInt = { Mux(isWrite(cmd), tileExclusiveDirty, state) } - def newStateOnCacheControl(cmd: Bits) = { + def newStateOnCacheControl(cmd: UInt) = { MuxLookup(cmd, tileInvalid, Array( M_INV -> tileInvalid, M_CLN -> tileShared @@ -502,7 +502,7 @@ class MSICoherence extends CoherencePolicyWithUncached { } def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix = { + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt = { MuxLookup(incoming.g_type, tileInvalid, Array( grantReadShared -> tileShared, grantReadExclusive -> tileExclusiveDirty, @@ -514,7 +514,7 @@ class MSICoherence extends CoherencePolicyWithUncached { grantAtomicUncached -> tileInvalid )) } - def newStateOnProbe(incoming: Probe, state: UFix): Bits = { + def newStateOnProbe(incoming: Probe, state: UInt): Bits = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeDowngrade -> tileShared, @@ -531,15 +531,15 @@ class MSICoherence extends CoherencePolicyWithUncached { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) } - def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { + def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, outstanding.a_type) } - def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = { + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -554,21 +554,21 @@ class MSICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uFixListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uSIntListContains(hasDataReleaseTypeList, rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getGrantType(a_type: UFix, count: UFix): Bits = { + def getGrantType(a_type: UInt, count: UInt): Bits = { MuxLookup(a_type, grantReadUncached, Array( - acquireReadShared -> Mux(count > UFix(0), grantReadShared, grantReadExclusive), + acquireReadShared -> Mux(count > UInt(0), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -577,14 +577,14 @@ class MSICoherence extends CoherencePolicyWithUncached { acquireAtomicUncached -> grantAtomicUncached )) } - def getGrantType(rel: Release, count: UFix): Bits = { + def getGrantType(rel: Release, count: UInt): Bits = { MuxLookup(rel.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) } - def getProbeType(a_type: UFix, global_state: UFix): UFix = { + def getProbeType(a_type: UInt, global_state: UInt): UInt = { MuxLookup(a_type, probeCopy, Array( acquireReadShared -> probeDowngrade, acquireReadExclusive -> probeInvalidate, @@ -593,20 +593,20 @@ class MSICoherence extends CoherencePolicyWithUncached { )) } - def needsOuterRead(a_type: UFix, global_state: UFix): Bool = { + def needsOuterRead(a_type: UInt, global_state: UInt): Bool = { (a_type != acquireWriteUncached) } - def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = { + def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def needsAckReply(a_type: UFix, global_state: UFix): Bool = { + def needsAckReply(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck def requiresAck(release: Release) = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) - def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) + def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } class MESICoherence extends CoherencePolicyWithUncached { @@ -617,45 +617,45 @@ class MESICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 7 def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UFix() } - val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(nMasterStates){ UFix() } + val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UInt() } + val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(nMasterStates){ UInt() } - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UFix() } - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UFix() } - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UFix() } + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UInt() } + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UInt() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UInt() } + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UInt() } val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) - def isHit (cmd: Bits, state: UFix): Bool = { + def isHit (cmd: UInt, state: UInt): Bool = { Mux(isWriteIntent(cmd), (state === tileExclusiveClean || state === tileExclusiveDirty), (state === tileShared || state === tileExclusiveClean || state === tileExclusiveDirty)) } - def isValid (state: UFix): Bool = { + def isValid (state: UInt): Bool = { state != tileInvalid } - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && messageIsUncached(outstanding)) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) } - def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { + def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( M_INV -> (state === tileExclusiveDirty), M_CLN -> (state === tileExclusiveDirty) )) } - def needsWriteback (state: UFix): Bool = { + def needsWriteback (state: UInt): Bool = { needsTransactionOnCacheControl(M_INV, state) } - def newStateOnHit(cmd: Bits, state: UFix): UFix = { + def newStateOnHit(cmd: UInt, state: UInt): UInt = { Mux(isWrite(cmd), tileExclusiveDirty, state) } - def newStateOnCacheControl(cmd: Bits) = { + def newStateOnCacheControl(cmd: UInt) = { MuxLookup(cmd, tileInvalid, Array( M_INV -> tileInvalid, M_CLN -> tileShared @@ -663,7 +663,7 @@ class MESICoherence extends CoherencePolicyWithUncached { } def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix = { + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt = { MuxLookup(incoming.g_type, tileInvalid, Array( grantReadShared -> tileShared, grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusive, tileExclusiveDirty, tileExclusiveClean), @@ -675,7 +675,7 @@ class MESICoherence extends CoherencePolicyWithUncached { grantAtomicUncached -> tileInvalid )) } - def newStateOnProbe(incoming: Probe, state: UFix): Bits = { + def newStateOnProbe(incoming: Probe, state: UInt): Bits = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeDowngrade -> tileShared, @@ -692,15 +692,15 @@ class MESICoherence extends CoherencePolicyWithUncached { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) } - def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { + def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, outstanding.a_type) } - def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = { + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -715,21 +715,21 @@ class MESICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uFixListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uSIntListContains(hasDataReleaseTypeList, rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getGrantType(a_type: UFix, count: UFix): Bits = { + def getGrantType(a_type: UInt, count: UInt): Bits = { MuxLookup(a_type, grantReadUncached, Array( - acquireReadShared -> Mux(count > UFix(0), grantReadShared, grantReadExclusive), + acquireReadShared -> Mux(count > UInt(0), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -738,14 +738,14 @@ class MESICoherence extends CoherencePolicyWithUncached { acquireAtomicUncached -> grantAtomicUncached )) } - def getGrantType(rel: Release, count: UFix): Bits = { + def getGrantType(rel: Release, count: UInt): Bits = { MuxLookup(rel.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) } - def getProbeType(a_type: UFix, global_state: UFix): UFix = { + def getProbeType(a_type: UInt, global_state: UInt): UInt = { MuxLookup(a_type, probeCopy, Array( acquireReadShared -> probeDowngrade, acquireReadExclusive -> probeInvalidate, @@ -757,13 +757,13 @@ class MESICoherence extends CoherencePolicyWithUncached { )) } - def needsOuterRead(a_type: UFix, global_state: UFix): Bool = { + def needsOuterRead(a_type: UInt, global_state: UInt): Bool = { (a_type != acquireWriteUncached) } - def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = { + def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def needsAckReply(a_type: UFix, global_state: UFix): Bool = { + def needsAckReply(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } @@ -771,7 +771,7 @@ class MESICoherence extends CoherencePolicyWithUncached { def requiresAck(release: Release) = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) - def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) + def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } class MigratoryCoherence extends CoherencePolicyWithUncached { @@ -782,45 +782,45 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 11 def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: tileSharedByTwo :: tileMigratoryClean :: tileMigratoryDirty :: Nil = Enum(nClientStates){ UFix() } + val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: tileSharedByTwo :: tileMigratoryClean :: tileMigratoryDirty :: Nil = Enum(nClientStates){ UInt() } - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(nAcquireTypes){ UFix() } - val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(nProbeTypes){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(nReleaseTypes){ UFix() } - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(nGrantTypes){ UFix() } + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(nAcquireTypes){ UInt() } + val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(nProbeTypes){ UInt() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(nReleaseTypes){ UInt() } + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(nGrantTypes){ UInt() } val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadMigratory, grantReadWordUncached, grantAtomicUncached) val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) - def isHit (cmd: Bits, state: UFix): Bool = { - Mux(isWriteIntent(cmd), uFixListContains(List(tileExclusiveClean, tileExclusiveDirty, tileMigratoryClean, tileMigratoryDirty), state), (state != tileInvalid)) + def isHit (cmd: UInt, state: UInt): Bool = { + Mux(isWriteIntent(cmd), uSIntListContains(List(tileExclusiveClean, tileExclusiveDirty, tileMigratoryClean, tileMigratoryDirty), state), (state != tileInvalid)) } - def isValid (state: UFix): Bool = { + def isValid (state: UInt): Bool = { state != tileInvalid } - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && messageIsUncached(outstanding)) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive && outstanding.a_type != acquireInvalidateOthers)) } - def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { + def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( - M_INV -> uFixListContains(List(tileExclusiveDirty,tileMigratoryDirty),state), - M_CLN -> uFixListContains(List(tileExclusiveDirty,tileMigratoryDirty),state) + M_INV -> uSIntListContains(List(tileExclusiveDirty,tileMigratoryDirty),state), + M_CLN -> uSIntListContains(List(tileExclusiveDirty,tileMigratoryDirty),state) )) } - def needsWriteback (state: UFix): Bool = { + def needsWriteback (state: UInt): Bool = { needsTransactionOnCacheControl(M_INV, state) } - def newStateOnHit(cmd: Bits, state: UFix): UFix = { + def newStateOnHit(cmd: UInt, state: UInt): UInt = { Mux(isWrite(cmd), MuxLookup(state, tileExclusiveDirty, Array( tileExclusiveClean -> tileExclusiveDirty, tileMigratoryClean -> tileMigratoryDirty)), state) } - def newStateOnCacheControl(cmd: Bits) = { + def newStateOnCacheControl(cmd: UInt) = { MuxLookup(cmd, tileInvalid, Array( M_INV -> tileInvalid, M_CLN -> tileShared @@ -828,7 +828,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix = { + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt = { MuxLookup(incoming.g_type, tileInvalid, Array( grantReadShared -> tileShared, grantReadExclusive -> MuxLookup(outstanding.a_type, tileExclusiveDirty, Array( @@ -846,7 +846,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { acquireReadShared -> tileMigratoryClean)) )) } - def newStateOnProbe(incoming: Probe, state: UFix): Bits = { + def newStateOnProbe(incoming: Probe, state: UInt): Bits = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeInvalidateOthers -> tileInvalid, @@ -869,17 +869,17 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = { Mux(isWriteIntent(cmd), Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), acquireReadShared) } - def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { + def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), outstanding.a_type) } - def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = { + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> Mux(uFixListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), + probeInvalidate -> Mux(uSIntListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), releaseInvalidateDataMigratory, releaseInvalidateData), probeDowngrade -> Mux(state === tileMigratoryDirty, releaseDowngradeDataMigratory, releaseDowngradeData), probeCopy -> releaseCopyData @@ -894,21 +894,21 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uFixListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uSIntListContains(hasDataReleaseTypeList, rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { - uFixListContains(List(grantReadShared, grantReadExclusive, grantReadMigratory), reply.g_type) + uSIntListContains(List(grantReadShared, grantReadExclusive, grantReadMigratory), reply.g_type) } - def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getGrantType(a_type: UFix, count: UFix): Bits = { + def getGrantType(a_type: UInt, count: UInt): Bits = { MuxLookup(a_type, grantReadUncached, Array( - acquireReadShared -> Mux(count > UFix(0), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? + acquireReadShared -> Mux(count > UInt(0), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -918,14 +918,14 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI? )) } - def getGrantType(rel: Release, count: UFix): Bits = { + def getGrantType(rel: Release, count: UInt): Bits = { MuxLookup(rel.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) } - def getProbeType(a_type: UFix, global_state: UFix): UFix = { + def getProbeType(a_type: UInt, global_state: UInt): UInt = { MuxLookup(a_type, probeCopy, Array( acquireReadShared -> probeDowngrade, acquireReadExclusive -> probeInvalidate, @@ -938,18 +938,18 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { )) } - def needsOuterRead(a_type: UFix, global_state: UFix): Bool = { + def needsOuterRead(a_type: UInt, global_state: UInt): Bool = { (a_type != acquireWriteUncached && a_type != acquireInvalidateOthers) } - def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = { + def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached || a_type === acquireAtomicUncached) } - def needsAckReply(a_type: UFix, global_state: UFix): Bool = { + def needsAckReply(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached ||a_type === acquireInvalidateOthers) } def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck def requiresAck(release: Release) = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) - def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) + def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index cb6465c5..3e7c2cd5 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -1,17 +1,17 @@ package uncore import Chisel._ -class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component +class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UInt])(gen: => T) extends Module { class Inputs extends Bundle { - val addr = UFix(INPUT, log2Up(n)) + val addr = UInt(INPUT, log2Up(n)) val rw = Bool(INPUT) val wdata = gen.asInput val wmask = gen.asInput override def clone = new Inputs().asInstanceOf[this.type] } val io = new Bundle { - val in = new PipeIO()(new Inputs).flip + val in = Valid(new Inputs).flip val rdata = gen.asOutput } val data = gen @@ -21,27 +21,27 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[Bit if (nDeep > 1 || colMux > 1) require(isPow2(n) && isPow2(leaf.n)) - val rdataDeep = Vec(nDeep) { Bits() } - val rdataSel = Vec(nDeep) { Bool() } + val rdataDeep = Vec.fill(nDeep){Bits()} + val rdataSel = Vec.fill(nDeep){Bool()} for (i <- 0 until nDeep) { - val in = Pipe(io.in.valid && (if (nDeep == 1) Bool(true) else UFix(i) === io.in.bits.addr(log2Up(n)-1, log2Up(n/nDeep))), io.in.bits, preLatency) + val in = Pipe(io.in.valid && (if (nDeep == 1) Bool(true) else UInt(i) === io.in.bits.addr(log2Up(n)-1, log2Up(n/nDeep))), io.in.bits, preLatency) val idx = in.bits.addr(log2Up(n/nDeep/colMux)-1, 0) val wdata = in.bits.wdata.toBits val wmask = in.bits.wmask.toBits val ren = in.valid && !in.bits.rw - val reg_ren = Reg(ren) - val rdata = Vec(nWide) { Bits() } + val reg_ren = RegUpdate(ren) + val rdata = Vec.fill(nWide){Bits()} val r = Pipe(ren, in.bits.addr, postLatency) for (j <- 0 until nWide) { val mem = leaf.clone var dout: Bits = null - val ridx = if (postLatency > 0) Reg() { Bits() } else null + val ridx = if (postLatency > 0) Reg(Bits()) else null var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) if (colMux > 1) - wmask0 = wmask0 & FillInterleaved(gen.width, UFixToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) + wmask0 = wmask0 & FillInterleaved(gen.width, UIntToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) when (in.valid) { when (in.bits.rw) { mem.write(idx, wdata0, wmask0) } @@ -61,7 +61,7 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[Bit var colMuxOut = rdataWide if (colMux > 1) { - val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.width*(k+1)-1, gen.width*k))) { Bits() } + val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.width*(k+1)-1, gen.width*k))) colMuxOut = colMuxIn(r.bits(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) } @@ -74,47 +74,49 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[Bit class LLCDataReq(ways: Int) extends MemReqCmd { - val way = UFix(width = log2Up(ways)) + val way = UInt(width = log2Up(ways)) val isWriteback = Bool() - override def clone = new LLCDataReq(ways).asInstanceOf[this.type] } -class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component +class LLCTagReq(ways: Int) extends HasMemAddr +{ + val way = UInt(width = log2Up(ways)) + override def clone = new LLCTagReq(ways).asInstanceOf[this.type] +} + +class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Module { val io = new Bundle { - val cpu = (new FIFOIO) { new MemReqCmd }.flip - val repl_way = UFix(INPUT, log2Up(ways)) + val cpu = Decoupled(new MemReqCmd).flip + val repl_way = UInt(INPUT, log2Up(ways)) val repl_dirty = Bool(INPUT) - val repl_tag = UFix(INPUT, MEM_ADDR_BITS - log2Up(sets)) - val data = (new FIFOIO) { new LLCDataReq(ways) } - val tag = (new FIFOIO) { new Bundle { - val addr = UFix(width = MEM_ADDR_BITS) - val way = UFix(width = log2Up(ways)) - } } + val repl_tag = UInt(INPUT, MEM_ADDR_BITS - log2Up(sets)) + val data = Decoupled(new LLCDataReq(ways)) + val tag = Decoupled(new LLCTagReq(ways)) val mem = new ioMemPipe - val mem_resp_set = UFix(OUTPUT, log2Up(sets)) - val mem_resp_way = UFix(OUTPUT, log2Up(ways)) + val mem_resp_set = UInt(OUTPUT, log2Up(sets)) + val mem_resp_way = UInt(OUTPUT, log2Up(ways)) } class MSHR extends Bundle { - val addr = UFix(width = PADDR_BITS - OFFSET_BITS) - val way = UFix(width = log2Up(ways)) + val addr = UInt(width = PADDR_BITS - OFFSET_BITS) + val way = UInt(width = log2Up(ways)) val tag = io.cpu.bits.tag.clone val refilled = Bool() - val refillCount = UFix(width = log2Up(REFILL_CYCLES)) + val refillCount = UInt(width = log2Up(REFILL_CYCLES)) val requested = Bool() val old_dirty = Bool() - val old_tag = UFix(width = MEM_ADDR_BITS - log2Up(sets)) + val old_tag = UInt(width = MEM_ADDR_BITS - log2Up(sets)) val wb_busy = Bool() override def clone = new MSHR().asInstanceOf[this.type] } - val valid = Vec(outstanding) { Reg(resetVal = Bool(false)) } + val valid = Vec.fill(outstanding){RegReset(Bool(false))} val validBits = valid.toBits val freeId = PriorityEncoder(~validBits) - val mshr = Vec(outstanding) { Reg() { new MSHR } } + val mshr = Vec.fill(outstanding){Reg(new MSHR)} when (io.cpu.valid && io.cpu.ready) { valid(freeId) := Bool(true) mshr(freeId).addr := io.cpu.bits.addr @@ -124,7 +126,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component mshr(freeId).old_tag := io.repl_tag mshr(freeId).wb_busy := Bool(false) mshr(freeId).requested := Bool(false) - mshr(freeId).refillCount := UFix(0) + mshr(freeId).refillCount := UInt(0) mshr(freeId).refilled := Bool(false) } @@ -136,8 +138,8 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val refillId = io.mem.resp.bits.tag(log2Up(outstanding)-1, 0) val refillCount = mshr(refillId).refillCount when (io.mem.resp.valid) { - mshr(refillId).refillCount := refillCount + UFix(1) - when (refillCount === UFix(REFILL_CYCLES-1)) { mshr(refillId).refilled := Bool(true) } + mshr(refillId).refillCount := refillCount + UInt(1) + when (refillCount === UInt(REFILL_CYCLES-1)) { mshr(refillId).refilled := Bool(true) } } val replays = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).refilled):_*) @@ -161,7 +163,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component io.data.bits.rw := Bool(false) io.data.bits.tag := mshr(replayId).tag io.data.bits.isWriteback := Bool(true) - io.data.bits.addr := Cat(mshr(writebackId).old_tag, mshr(writebackId).addr(log2Up(sets)-1, 0)).toUFix + io.data.bits.addr := Cat(mshr(writebackId).old_tag, mshr(writebackId).addr(log2Up(sets)-1, 0)).toUInt io.data.bits.way := mshr(writebackId).way when (replay) { io.data.valid := io.tag.ready @@ -181,25 +183,25 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component io.mem_resp_way := mshr(refillId).way } -class LLCWriteback(requestors: Int) extends Component +class LLCWriteback(requestors: Int) extends Module { val io = new Bundle { - val req = Vec(requestors) { (new FIFOIO) { UFix(width = MEM_ADDR_BITS) }.flip } - val data = Vec(requestors) { (new FIFOIO) { new MemData }.flip } + val req = Vec.fill(requestors){Decoupled(UInt(width = MEM_ADDR_BITS)).flip } + val data = Vec.fill(requestors){Decoupled(new MemData).flip } val mem = new ioMemPipe } - val valid = Reg(resetVal = Bool(false)) - val who = Reg() { UFix() } - val addr = Reg() { UFix() } - val cmd_sent = Reg() { Bool() } - val data_sent = Reg() { Bool() } - val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) + val valid = RegReset(Bool(false)) + val who = Reg(UInt()) + val addr = Reg(UInt()) + val cmd_sent = Reg(Bool()) + val data_sent = Reg(Bool()) + val count = RegReset(UInt(0, log2Up(REFILL_CYCLES))) var anyReq = Bool(false) for (i <- 0 until requestors) { io.req(i).ready := !valid && !anyReq - io.data(i).ready := valid && who === UFix(i) && io.mem.req_data.ready + io.data(i).ready := valid && who === UInt(i) && io.mem.req_data.ready anyReq = anyReq || io.req(i).valid } @@ -213,8 +215,8 @@ class LLCWriteback(requestors: Int) extends Component } when (io.mem.req_data.valid && io.mem.req_data.ready) { - count := count + UFix(1) - when (count === UFix(REFILL_CYCLES-1)) { + count := count + UInt(1) + when (count === UInt(REFILL_CYCLES-1)) { data_sent := Bool(true) when (cmd_sent) { valid := Bool(false) } } @@ -230,51 +232,51 @@ class LLCWriteback(requestors: Int) extends Component io.mem.req_data.bits := io.data(who).bits } -class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Component +class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[UInt]) extends Module { val io = new Bundle { - val req = (new FIFOIO) { new LLCDataReq(ways) }.flip - val req_data = (new FIFOIO) { new MemData }.flip - val writeback = (new FIFOIO) { UFix(width = MEM_ADDR_BITS) } - val writeback_data = (new FIFOIO) { new MemData } - val resp = (new FIFOIO) { new MemResp } - val mem_resp = (new PipeIO) { new MemResp }.flip - val mem_resp_set = UFix(INPUT, log2Up(sets)) - val mem_resp_way = UFix(INPUT, log2Up(ways)) + val req = Decoupled(new LLCDataReq(ways)).flip + val req_data = Decoupled(new MemData).flip + val writeback = Decoupled(UInt(width = MEM_ADDR_BITS)) + val writeback_data = Decoupled(new MemData) + val resp = Decoupled(new MemResp) + val mem_resp = Valid(new MemResp).flip + val mem_resp_set = UInt(INPUT, log2Up(sets)) + val mem_resp_way = UInt(INPUT, log2Up(ways)) } - val data = new BigMem(sets*ways*REFILL_CYCLES, 1, latency-1, leaf)(Bits(width = MEM_DATA_BITS)) + val data = Module(new BigMem(sets*ways*REFILL_CYCLES, 1, latency-1, leaf)(Bits(width = MEM_DATA_BITS))) class QEntry extends MemResp { val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] } - val q = (new Queue(latency+2)) { new QEntry } - val qReady = q.io.count <= UFix(q.entries-latency-1) - val valid = Reg(resetVal = Bool(false)) - val req = Reg() { io.req.bits.clone } - val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - val refillCount = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) + val q = Module(new Queue(new QEntry, latency+2)) + val qReady = q.io.count <= UInt(q.entries-latency-1) + val valid = RegReset(Bool(false)) + val req = Reg(io.req.bits.clone) + val count = RegReset(UInt(0, log2Up(REFILL_CYCLES))) + val refillCount = RegReset(UInt(0, log2Up(REFILL_CYCLES))) when (data.io.in.valid && !io.mem_resp.valid) { - count := count + UFix(1) - when (valid && count === UFix(REFILL_CYCLES-1)) { valid := Bool(false) } + count := count + UInt(1) + when (valid && count === UInt(REFILL_CYCLES-1)) { valid := Bool(false) } } when (io.req.valid && io.req.ready) { valid := Bool(true); req := io.req.bits } - when (io.mem_resp.valid) { refillCount := refillCount + UFix(1) } + when (io.mem_resp.valid) { refillCount := refillCount + UInt(1) } data.io.in.valid := io.req.valid && io.req.ready && Mux(io.req.bits.rw, io.req_data.valid, qReady) - data.io.in.bits.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUFix + data.io.in.bits.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUInt data.io.in.bits.rw := io.req.bits.rw data.io.in.bits.wdata := io.req_data.bits.data - data.io.in.bits.wmask := Fix(-1, io.req_data.bits.data.width) + data.io.in.bits.wmask := SInt(-1, io.req_data.bits.data.width) when (valid) { data.io.in.valid := Mux(req.rw, io.req_data.valid, qReady) - data.io.in.bits.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUFix + data.io.in.bits.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUInt data.io.in.bits.rw := req.rw } when (io.mem_resp.valid) { data.io.in.valid := Bool(true) - data.io.in.bits.addr := Cat(io.mem_resp_way, io.mem_resp_set, refillCount).toUFix + data.io.in.bits.addr := Cat(io.mem_resp_way, io.mem_resp_set, refillCount).toUInt data.io.in.bits.rw := Bool(true) data.io.in.bits.wdata := io.mem_resp.bits.data } @@ -298,21 +300,21 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Compo io.writeback_data.bits := q.io.deq.bits } -class MemReqArb(n: Int) extends Component // UNTESTED +class MemReqArb(n: Int) extends Module // UNTESTED { val io = new Bundle { - val cpu = Vec(n) { new ioMem().flip } + val cpu = Vec.fill(n){new ioMem().flip} val mem = new ioMem } - val lock = Reg(resetVal = Bool(false)) - val locker = Reg() { UFix() } + val lock = RegReset(Bool(false)) + val locker = Reg(UInt()) - val arb = new RRArbiter(n)(new MemReqCmd) + val arb = Module(new RRArbiter(new MemReqCmd, n)) val respWho = io.mem.resp.bits.tag(log2Up(n)-1,0) - val respTag = io.mem.resp.bits.tag >> UFix(log2Up(n)) + val respTag = io.mem.resp.bits.tag >> UInt(log2Up(n)) for (i <- 0 until n) { - val me = UFix(i, log2Up(n)) + val me = UInt(i, log2Up(n)) arb.io.in(i).valid := io.cpu(i).req_cmd.valid arb.io.in(i).bits := io.cpu(i).req_cmd.bits arb.io.in(i).bits.tag := Cat(io.cpu(i).req_cmd.bits.tag, me) @@ -323,7 +325,7 @@ class MemReqArb(n: Int) extends Component // UNTESTED val haveLock = lock && locker === me when (getLock) { lock := Bool(true) - locker := UFix(i) + locker := UInt(i) } when (getLock || haveLock) { io.cpu(i).req_data.ready := io.mem.req_data.ready @@ -341,7 +343,7 @@ class MemReqArb(n: Int) extends Component // UNTESTED when (unlock) { lock := Bool(false) } } -class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits]) extends Component +class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[UInt], dataLeaf: Mem[UInt]) extends Module { val io = new Bundle { val cpu = new ioMem().flip @@ -351,48 +353,48 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val tagWidth = MEM_ADDR_BITS - log2Up(sets) val metaWidth = tagWidth + 2 // valid + dirty - val memCmdArb = (new Arbiter(2)) { new MemReqCmd } - val dataArb = (new Arbiter(2)) { new LLCDataReq(ways) } - val mshr = new LLCMSHRFile(sets, ways, outstanding) - val tags = new BigMem(sets, 0, 1, tagLeaf)(Bits(width = metaWidth*ways)) - val data = new LLCData(4, sets, ways, dataLeaf) - val writeback = new LLCWriteback(2) + val memCmdArb = Module(new Arbiter(new MemReqCmd, 2)) + val dataArb = Module(new Arbiter(new LLCDataReq(ways), 2)) + val mshr = Module(new LLCMSHRFile(sets, ways, outstanding)) + val tags = Module(new BigMem(sets, 0, 1, tagLeaf)(Bits(width = metaWidth*ways))) + val data = Module(new LLCData(4, sets, ways, dataLeaf)) + val writeback = Module(new LLCWriteback(2)) - val initCount = Reg(resetVal = UFix(0, log2Up(sets+1))) + val initCount = RegReset(UInt(0, log2Up(sets+1))) val initialize = !initCount(log2Up(sets)) - when (initialize) { initCount := initCount + UFix(1) } + when (initialize) { initCount := initCount + UInt(1) } - val replay_s2 = Reg(resetVal = Bool(false)) - val s2_valid = Reg(resetVal = Bool(false)) - val s2 = Reg() { new MemReqCmd } + val replay_s2 = RegReset(Bool(false)) + val s2_valid = RegReset(Bool(false)) + val s2 = Reg(new MemReqCmd) val s3_rdy = Bool() val replay_s2_rdy = Bool() - val s1_valid = Reg(io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy, resetVal = Bool(false)) - val s1 = Reg() { new MemReqCmd } + val s1_valid = Reg(update = io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy, reset = Bool(false)) + val s1 = Reg(new MemReqCmd) when (io.cpu.req_cmd.fire()) { s1 := io.cpu.req_cmd.bits } when (replay_s2 && replay_s2_rdy) { s1 := s2 } s2_valid := s1_valid replay_s2 := s2_valid && !s3_rdy || replay_s2 && !replay_s2_rdy - val s2_tags = Vec(ways) { Reg() { Bits(width = metaWidth) } } + val s2_tags = Vec.fill(ways){Reg(Bits(width = metaWidth))} when (s1_valid) { s2 := s1 for (i <- 0 until ways) s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) } val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.width-1, s2.addr.width-tagWidth) === t(tagWidth-1, 0)) - val s2_hit_way = OHToUFix(s2_hits) + val s2_hit_way = OHToUInt(s2_hits) val s2_hit = s2_hits.reduceLeft(_||_) val s2_hit_dirty = s2_tags(s2_hit_way)(tagWidth+1) val repl_way = LFSR16(s2_valid)(log2Up(ways)-1, 0) - val repl_tag = s2_tags(repl_way).toUFix + val repl_tag = s2_tags(repl_way).toUInt val setDirty = s2_valid && s2.rw && s2_hit && !s2_hit_dirty val tag_we = initialize || setDirty || mshr.io.tag.fire() val tag_waddr = Mux(initialize, initCount, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)) val tag_wdata = Cat(setDirty, !initialize, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)) - val tag_wmask = Mux(initialize, Fix(-1, ways), UFixToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way))) + val tag_wmask = Mux(initialize, SInt(-1, ways), UIntToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way))) tags.io.in.valid := io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy || tag_we tags.io.in.bits.addr := Mux(tag_we, tag_waddr, Mux(replay_s2, s2.addr, io.cpu.req_cmd.bits.addr)(log2Up(sets)-1,0)) tags.io.in.bits.rw := tag_we @@ -440,16 +442,16 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da io.mem.req_data <> writeback.io.mem.req_data } -class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Component +class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module { - val io = new ioQueue(entries)(data) + val io = new QueueIO(data, entries) require(isPow2(entries) && entries > 1) val do_flow = Bool() val do_enq = io.enq.fire() && !do_flow val do_deq = io.deq.fire() && !do_flow - val maybe_full = Reg(resetVal = Bool(false)) + val maybe_full = RegReset(Bool(false)) val enq_ptr = Counter(do_enq, entries)._1 val deq_ptr = Counter(do_deq, entries)._1 when (do_enq != do_deq) { maybe_full := do_enq } @@ -457,37 +459,37 @@ class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Component val ptr_match = enq_ptr === deq_ptr val empty = ptr_match && !maybe_full val full = ptr_match && maybe_full - val atLeastTwo = full || enq_ptr - deq_ptr >= UFix(2) + val atLeastTwo = full || enq_ptr - deq_ptr >= UInt(2) do_flow := empty && io.deq.ready - val ram = Mem(entries, seqRead = true){Bits(width = data.getWidth)} - val ram_addr = Reg{Bits()} - val ram_out_valid = Reg{Bool()} + val ram = Mem(data, entries, seqRead = true) + val ram_addr = Reg(Bits()) + val ram_out_valid = Reg(Bool()) ram_out_valid := Bool(false) when (do_enq) { ram(enq_ptr) := io.enq.bits.toBits } when (io.deq.ready && (atLeastTwo || !io.deq.valid && !empty)) { ram_out_valid := Bool(true) - ram_addr := Mux(io.deq.valid, deq_ptr + UFix(1), deq_ptr) + ram_addr := Mux(io.deq.valid, deq_ptr + UInt(1), deq_ptr) } io.deq.valid := Mux(empty, io.enq.valid, ram_out_valid) io.enq.ready := !full - io.deq.bits := Mux(empty, io.enq.bits, data.fromBits(ram(ram_addr))) + io.deq.bits := Mux(empty, io.enq.bits, ram(ram_addr)) } -class HellaQueue[T <: Data](val entries: Int)(data: => T) extends Component +class HellaQueue[T <: Data](val entries: Int)(data: => T) extends Module { - val io = new ioQueue(entries)(data) + val io = new QueueIO(data, entries) - val fq = new HellaFlowQueue(entries)(data) + val fq = Module(new HellaFlowQueue(entries)(data)) io.enq <> fq.io.enq io.deq <> Queue(fq.io.deq, 1, pipe = true) } object HellaQueue { - def apply[T <: Data](enq: FIFOIO[T], entries: Int) = { - val q = (new HellaQueue(entries)) { enq.bits.clone } + def apply[T <: Data](enq: DecoupledIO[T], entries: Int) = { + val q = Module((new HellaQueue(entries)) { enq.bits.clone }) q.io.enq.valid := enq.valid // not using <> so that override is allowed q.io.enq.bits := enq.bits enq.ready := q.io.enq.ready @@ -495,7 +497,7 @@ object HellaQueue } } -class DRAMSideLLCNull(numRequests: Int, refillCycles: Int) extends Component +class DRAMSideLLCNull(numRequests: Int, refillCycles: Int) extends Module { val io = new Bundle { val cpu = new ioMem().flip @@ -507,17 +509,17 @@ class DRAMSideLLCNull(numRequests: Int, refillCycles: Int) extends Component val inc = Bool() val dec = Bool() - val count = Reg(resetVal = UFix(numEntries, size)) - val watermark = count >= UFix(refillCycles) + val count = RegReset(UInt(numEntries, size)) + val watermark = count >= UInt(refillCycles) when (inc && !dec) { - count := count + UFix(1) + count := count + UInt(1) } when (!inc && dec) { - count := count - UFix(refillCycles) + count := count - UInt(refillCycles) } when (inc && dec) { - count := count - UFix(refillCycles-1) + count := count - UInt(refillCycles-1) } val cmdq_mask = io.cpu.req_cmd.bits.rw || watermark @@ -528,7 +530,7 @@ class DRAMSideLLCNull(numRequests: Int, refillCycles: Int) extends Component io.mem.req_data <> io.cpu.req_data - val resp_dataq = (new HellaQueue(numEntries)) { new MemResp } + val resp_dataq = Module((new HellaQueue(numEntries)) { new MemResp }) resp_dataq.io.enq <> io.mem.resp io.cpu.resp <> resp_dataq.io.deq diff --git a/uncore/src/memserdes.scala b/uncore/src/memserdes.scala index b1272b28..9badb5e9 100644 --- a/uncore/src/memserdes.scala +++ b/uncore/src/memserdes.scala @@ -2,40 +2,45 @@ package uncore import Chisel._ import scala.math._ -class MemData extends Bundle { +trait HasMemData extends Bundle { val data = Bits(width = MEM_DATA_BITS) } -class MemReqCmd extends Bundle { - val addr = UFix(width = MEM_ADDR_BITS) +trait HasMemAddr extends Bundle { + val addr = UInt(width = MEM_ADDR_BITS) +} + +trait HasMemTag extends Bundle { + val tag = UInt(width = MEM_TAG_BITS) +} + +class MemReqCmd extends HasMemAddr with HasMemTag { val rw = Bool() - val tag = Bits(width = MEM_TAG_BITS) } -class MemResp extends Bundle { - val tag = Bits(width = MEM_TAG_BITS) - val data = Bits(width = MEM_DATA_BITS) -} +class MemResp extends HasMemData with HasMemTag + +class MemData extends HasMemData class ioMem extends Bundle { - val req_cmd = (new FIFOIO) { new MemReqCmd() } - val req_data = (new FIFOIO) { new MemData() } - val resp = (new FIFOIO) { new MemResp() }.flip + val req_cmd = Decoupled(new MemReqCmd) + val req_data = Decoupled(new MemData) + val resp = Decoupled(new MemResp).flip } class ioMemPipe extends Bundle { - val req_cmd = (new FIFOIO) { new MemReqCmd() } - val req_data = (new FIFOIO) { new MemData() } - val resp = (new PipeIO) { new MemResp() }.flip + val req_cmd = Decoupled(new MemReqCmd) + val req_data = Decoupled(new MemData) + val resp = Valid(new MemResp).flip } class ioMemSerialized(w: Int) extends Bundle { - val req = (new FIFOIO) { Bits(width = w) } - val resp = (new PipeIO) { Bits(width = w) }.flip + val req = Decoupled(Bits(width = w)) + val resp = Valid(Bits(width = w)).flip } -class MemSerdes(w: Int) extends Component +class MemSerdes(w: Int) extends Module { val io = new Bundle { val wide = new ioMem().flip @@ -45,19 +50,19 @@ class MemSerdes(w: Int) extends Component val dbits = io.wide.req_data.bits.toBits.getWidth val rbits = io.wide.resp.bits.getWidth - val out_buf = Reg() { Bits() } - val in_buf = Reg() { Bits() } + val out_buf = Reg(Bits()) + val in_buf = Reg(Bits()) - val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(5) { UFix() } - val state = Reg(resetVal = s_idle) - val send_cnt = Reg(resetVal = UFix(0, log2Up((max(abits, dbits)+w-1)/w))) - val data_send_cnt = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - val adone = io.narrow.req.ready && send_cnt === UFix((abits-1)/w) - val ddone = io.narrow.req.ready && send_cnt === UFix((dbits-1)/w) + val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(5) { UInt() } + val state = RegReset(s_idle) + val send_cnt = RegReset(UInt(0, log2Up((max(abits, dbits)+w-1)/w))) + val data_send_cnt = RegReset(UInt(0, log2Up(REFILL_CYCLES))) + val adone = io.narrow.req.ready && send_cnt === UInt((abits-1)/w) + val ddone = io.narrow.req.ready && send_cnt === UInt((dbits-1)/w) when (io.narrow.req.valid && io.narrow.req.ready) { - send_cnt := send_cnt + UFix(1) - out_buf := out_buf >> UFix(w) + send_cnt := send_cnt + UInt(1) + out_buf := out_buf >> UInt(w) } when (io.wide.req_cmd.valid && io.wide.req_cmd.ready) { out_buf := io.wide.req_cmd.bits.toBits @@ -76,31 +81,31 @@ class MemSerdes(w: Int) extends Component } when (state === s_read_addr && adone) { state := s_idle - send_cnt := UFix(0) + send_cnt := UInt(0) } when (state === s_write_addr && adone) { state := s_write_idle - send_cnt := UFix(0) + send_cnt := UInt(0) } when (state === s_write_idle && io.wide.req_data.valid) { state := s_write_data } when (state === s_write_data && ddone) { - data_send_cnt := data_send_cnt + UFix(1) - state := Mux(data_send_cnt === UFix(REFILL_CYCLES-1), s_idle, s_write_idle) - send_cnt := UFix(0) + data_send_cnt := data_send_cnt + UInt(1) + state := Mux(data_send_cnt === UInt(REFILL_CYCLES-1), s_idle, s_write_idle) + send_cnt := UInt(0) } - val recv_cnt = Reg(resetVal = UFix(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - val resp_val = Reg(resetVal = Bool(false)) + val recv_cnt = RegReset(UInt(0, log2Up((rbits+w-1)/w))) + val data_recv_cnt = RegReset(UInt(0, log2Up(REFILL_CYCLES))) + val resp_val = RegReset(Bool(false)) resp_val := Bool(false) when (io.narrow.resp.valid) { - recv_cnt := recv_cnt + UFix(1) - when (recv_cnt === UFix((rbits-1)/w)) { - recv_cnt := UFix(0) - data_recv_cnt := data_recv_cnt + UFix(1) + recv_cnt := recv_cnt + UInt(1) + when (recv_cnt === UInt((rbits-1)/w)) { + recv_cnt := UInt(0) + data_recv_cnt := data_recv_cnt + UInt(1) resp_val := Bool(true) } in_buf := Cat(io.narrow.resp.bits, in_buf((rbits+w-1)/w*w-1,w)) @@ -108,7 +113,7 @@ class MemSerdes(w: Int) extends Component io.wide.resp.valid := resp_val io.wide.resp.bits.tag := in_buf(io.wide.resp.bits.tag.width-1,0) - io.wide.resp.bits.data := in_buf >> UFix(io.wide.resp.bits.tag.width) + io.wide.resp.bits.data := in_buf >> UInt(io.wide.resp.bits.tag.width) } class MemDesserIO(w: Int) extends Bundle { @@ -116,7 +121,7 @@ class MemDesserIO(w: Int) extends Bundle { val wide = new ioMem } -class MemDesser(w: Int) extends Component // test rig side +class MemDesser(w: Int) extends Module // test rig side { val io = new MemDesserIO(w) val abits = io.wide.req_cmd.bits.toBits.getWidth @@ -124,59 +129,59 @@ class MemDesser(w: Int) extends Component // test rig side val rbits = io.wide.resp.bits.getWidth require(dbits >= abits && rbits >= dbits) - val recv_cnt = Reg(resetVal = UFix(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - val adone = io.narrow.req.valid && recv_cnt === UFix((abits-1)/w) - val ddone = io.narrow.req.valid && recv_cnt === UFix((dbits-1)/w) - val rdone = io.narrow.resp.valid && recv_cnt === UFix((rbits-1)/w) + val recv_cnt = RegReset(UInt(0, log2Up((rbits+w-1)/w))) + val data_recv_cnt = RegReset(UInt(0, log2Up(REFILL_CYCLES))) + val adone = io.narrow.req.valid && recv_cnt === UInt((abits-1)/w) + val ddone = io.narrow.req.valid && recv_cnt === UInt((dbits-1)/w) + val rdone = io.narrow.resp.valid && recv_cnt === UInt((rbits-1)/w) - val s_cmd_recv :: s_cmd :: s_data_recv :: s_data :: s_reply :: Nil = Enum(5) { UFix() } - val state = Reg(resetVal = s_cmd_recv) + val s_cmd_recv :: s_cmd :: s_data_recv :: s_data :: s_reply :: Nil = Enum(5) { UInt() } + val state = RegReset(s_cmd_recv) - val in_buf = Reg() { Bits() } + val in_buf = Reg(Bits()) when (io.narrow.req.valid && io.narrow.req.ready || io.narrow.resp.valid) { - recv_cnt := recv_cnt + UFix(1) + recv_cnt := recv_cnt + UInt(1) in_buf := Cat(io.narrow.req.bits, in_buf((rbits+w-1)/w*w-1,w)) } io.narrow.req.ready := state === s_cmd_recv || state === s_data_recv when (state === s_cmd_recv && adone) { state := s_cmd - recv_cnt := UFix(0) + recv_cnt := UInt(0) } when (state === s_cmd && io.wide.req_cmd.ready) { state := Mux(io.wide.req_cmd.bits.rw, s_data_recv, s_reply) } when (state === s_data_recv && ddone) { state := s_data - recv_cnt := UFix(0) + recv_cnt := UInt(0) } when (state === s_data && io.wide.req_data.ready) { state := s_data_recv - when (data_recv_cnt === UFix(REFILL_CYCLES-1)) { + when (data_recv_cnt === UInt(REFILL_CYCLES-1)) { state := s_cmd_recv } - data_recv_cnt := data_recv_cnt + UFix(1) + data_recv_cnt := data_recv_cnt + UInt(1) } when (rdone) { // state === s_reply - when (data_recv_cnt === UFix(REFILL_CYCLES-1)) { + when (data_recv_cnt === UInt(REFILL_CYCLES-1)) { state := s_cmd_recv } - recv_cnt := UFix(0) - data_recv_cnt := data_recv_cnt + UFix(1) + recv_cnt := UInt(0) + data_recv_cnt := data_recv_cnt + UInt(1) } - val req_cmd = in_buf >> UFix(((rbits+w-1)/w - (abits+w-1)/w)*w) + val req_cmd = in_buf >> UInt(((rbits+w-1)/w - (abits+w-1)/w)*w) io.wide.req_cmd.valid := state === s_cmd io.wide.req_cmd.bits := io.wide.req_cmd.bits.fromBits(req_cmd) io.wide.req_data.valid := state === s_data - io.wide.req_data.bits.data := in_buf >> UFix(((rbits+w-1)/w - (dbits+w-1)/w)*w) + io.wide.req_data.bits.data := in_buf >> UInt(((rbits+w-1)/w - (dbits+w-1)/w)*w) - val dataq = (new Queue(REFILL_CYCLES)) { new MemResp } + val dataq = Module(new Queue(new MemResp, REFILL_CYCLES)) dataq.io.enq <> io.wide.resp - dataq.io.deq.ready := recv_cnt === UFix((rbits-1)/w) + dataq.io.deq.ready := recv_cnt === UInt((rbits-1)/w) io.narrow.resp.valid := dataq.io.deq.valid - io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UFix(w)) + io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UInt(w)) } diff --git a/uncore/src/network.scala b/uncore/src/network.scala index cdaa8118..29c1dee8 100644 --- a/uncore/src/network.scala +++ b/uncore/src/network.scala @@ -3,24 +3,24 @@ import Chisel._ import scala.collection.mutable.Stack class PairedDataIO[M <: Data, D <: Data]()(m: => M, d: => D) extends Bundle { - val meta = new FIFOIO()(m) - val data = new FIFOIO()(d) + val meta = Decoupled(m) + val data = Decoupled(d) override def clone = { new PairedDataIO()(m,d).asInstanceOf[this.type] } } class PairedArbiterIO[M <: Data, D <: Data](n: Int)(m: => M, d: => D) extends Bundle { - val in = Vec(n) { new PairedDataIO()(m,d) }.flip + val in = Vec.fill(n){new PairedDataIO()(m,d)}.flip val out = new PairedDataIO()(m,d) val meta_chosen = Bits(OUTPUT, log2Up(n)) val data_chosen = Bits(OUTPUT, log2Up(n)) override def clone = { new PairedArbiterIO(n)(m,d).asInstanceOf[this.type] } } -class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock: Option[M => Bool] = None)(meta: => M, data: => D) extends Component { +class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock: Option[M => Bool] = None)(meta: => M, data: => D) extends Module { require(isPow2(count)) val io = new PairedArbiterIO(n)(meta,data) - val locked = if(count > 1) Reg(resetVal = Bool(false)) else Bool(false) - val lockIdx = if(count > 1) Reg(resetVal = UFix(n-1)) else UFix(n-1) + val locked = if(count > 1) RegReset(Bool(false)) else Bool(false) + val lockIdx = if(count > 1) RegReset(UInt(n-1)) else UInt(n-1) val grant = List.fill(n)(Bool()) val meta_chosen = Bits(width = log2Up(n)) @@ -28,7 +28,7 @@ class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock val valid_meta_has_data = io.in(meta_chosen).meta.valid && chosen_meta_has_data val grant_chosen_meta = !(locked && chosen_meta_has_data) (0 until n).map(i => io.in(i).meta.ready := grant(i) && grant_chosen_meta && io.out.meta.ready) - (0 until n).map(i => io.in(i).data.ready := Mux(locked, lockIdx === UFix(i), grant(i) && valid_meta_has_data) && io.out.data.ready) + (0 until n).map(i => io.in(i).data.ready := Mux(locked, lockIdx === UInt(i), grant(i) && valid_meta_has_data) && io.out.data.ready) io.out.meta.valid := io.in(meta_chosen).meta.valid && grant_chosen_meta io.out.data.valid := Mux(locked, io.in(lockIdx).data.valid, io.in(meta_chosen).data.valid && valid_meta_has_data) io.out.meta.bits := io.in(meta_chosen).meta.bits @@ -37,11 +37,11 @@ class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock io.data_chosen := Mux(locked, lockIdx, meta_chosen) if(count > 1){ - val cnt = Reg(resetVal = UFix(0, width = log2Up(count))) - val cnt_next = cnt + UFix(1) + val cnt = RegReset(UInt(0, width = log2Up(count))) + val cnt_next = cnt + UInt(1) when(io.out.data.fire()){ cnt := cnt_next - when(cnt_next === UFix(0)) { + when(cnt_next === UInt(0)) { locked := Bool(false) } } @@ -49,20 +49,20 @@ class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock when(needsLock.map(_(io.out.meta.bits)).getOrElse(Bool(true))) { when(!locked) { locked := Bool(true) - lockIdx := Vec(io.in.map{in => in.meta.fire()}){Bool()}.indexWhere{i: Bool => i} + lockIdx := Vec(io.in.map{in => in.meta.fire()}).indexWhere{i: Bool => i} } } } } - val last_grant = Reg(resetVal = Bits(0, log2Up(n))) - val ctrl = ArbiterCtrl((0 until n).map(i => io.in(i).meta.valid && UFix(i) > last_grant) ++ io.in.map(_.meta.valid)) - (0 until n).map(i => grant(i) := ctrl(i) && UFix(i) > last_grant || ctrl(i + n)) + val last_grant = RegReset(Bits(0, log2Up(n))) + val ctrl = ArbiterCtrl((0 until n).map(i => io.in(i).meta.valid && UInt(i) > last_grant) ++ io.in.map(_.meta.valid)) + (0 until n).map(i => grant(i) := ctrl(i) && UInt(i) > last_grant || ctrl(i + n)) var choose = Bits(n-1) for (i <- n-2 to 0 by -1) choose = Mux(io.in(i).meta.valid, Bits(i), choose) for (i <- n-1 to 1 by -1) - choose = Mux(io.in(i).meta.valid && UFix(i) > last_grant, Bits(i), choose) + choose = Mux(io.in(i).meta.valid && UInt(i) > last_grant, Bits(i), choose) meta_chosen := choose when (io.out.meta.fire()) { last_grant := meta_chosen } @@ -70,23 +70,23 @@ class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock class PairedCrossbar[M <: Data, D <: Data](count: Int, needsLock: Option[PhysicalNetworkIO[M] => Bool] = None)(meta: => M, data: => D)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { val io = new Bundle { - val in = Vec(conf.nEndpoints){new PairedDataIO()(new PhysicalNetworkIO()(meta),new PhysicalNetworkIO()(data))}.flip - val out = Vec(conf.nEndpoints){new PairedDataIO()(new PhysicalNetworkIO()(meta),new PhysicalNetworkIO()(data))} + val in = Vec.fill(conf.nEndpoints){new PairedDataIO()(new PhysicalNetworkIO()(meta),new PhysicalNetworkIO()(data))}.flip + val out = Vec.fill(conf.nEndpoints){new PairedDataIO()(new PhysicalNetworkIO()(meta),new PhysicalNetworkIO()(data))} } - val metaRdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) - val dataRdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) + val metaRdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints){Bool()}) + val dataRdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints){Bool()}) val rdyVecs = metaRdyVecs zip dataRdyVecs io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { - val rrarb = new PairedLockingRRArbiter(conf.nEndpoints, count, needsLock)(io.in(0).meta.bits.clone, io.in(0).data.bits.clone) + val rrarb = Module(new PairedLockingRRArbiter(conf.nEndpoints, count, needsLock)(io.in(0).meta.bits.clone, io.in(0).data.bits.clone)) rrarb.io.in zip io.in zip rdys._1 zip rdys._2 map { case (((arb, in), meta_rdy), data_rdy) => { - arb.meta.valid := in.meta.valid && (in.meta.bits.header.dst === UFix(i)) + arb.meta.valid := in.meta.valid && (in.meta.bits.header.dst === UInt(i)) arb.meta.bits := in.meta.bits - meta_rdy := arb.meta.ready && (in.meta.bits.header.dst === UFix(i)) - arb.data.valid := in.data.valid && (in.data.bits.header.dst === UFix(i)) + meta_rdy := arb.meta.ready && (in.meta.bits.header.dst === UInt(i)) + arb.data.valid := in.data.valid && (in.data.bits.header.dst === UInt(i)) arb.data.bits := in.data.bits - data_rdy := arb.data.ready && (in.data.bits.header.dst === UFix(i)) + data_rdy := arb.data.ready && (in.data.bits.header.dst === UInt(i)) }} out <> rrarb.io.out }} @@ -99,8 +99,8 @@ class PairedCrossbar[M <: Data, D <: Data](count: Int, needsLock: Option[Physica case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { - val src = UFix(width = conf.idBits) - val dst = UFix(width = conf.idBits) + val src = UInt(width = conf.idBits) + val dst = UInt(width = conf.idBits) } class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { @@ -109,22 +109,22 @@ class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkC override def clone = { new PhysicalNetworkIO()(data).asInstanceOf[this.type] } } -abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Component +abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Module class BasicCrossbar[T <: Data](count: Int = 1)(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { val io = new Bundle { - val in = Vec(conf.nEndpoints){(new FIFOIO){(new PhysicalNetworkIO){data}}}.flip - val out = Vec(conf.nEndpoints){(new FIFOIO){(new PhysicalNetworkIO){data}}} + val in = Vec.fill(conf.nEndpoints){Decoupled((new PhysicalNetworkIO){data})}.flip + val out = Vec.fill(conf.nEndpoints){Decoupled((new PhysicalNetworkIO){data})} } - val rdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) + val rdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints)(Bool())) io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { - val rrarb = (new LockingRRArbiter(conf.nEndpoints, count)){io.in(0).bits.clone} + val rrarb = Module(new LockingRRArbiter(io.in(0).bits, conf.nEndpoints, count)) (rrarb.io.in, io.in, rdys).zipped.map{ case (arb, in, rdy) => { - arb.valid := in.valid && (in.bits.header.dst === UFix(i)) + arb.valid := in.valid && (in.bits.header.dst === UInt(i)) arb.bits := in.bits - rdy := arb.ready && (in.bits.header.dst === UFix(i)) + rdy := arb.ready && (in.bits.header.dst === UInt(i)) }} out <> rrarb.io.out }} @@ -135,30 +135,30 @@ class BasicCrossbar[T <: Data](count: Int = 1)(data: => T)(implicit conf: Physic case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nMasters: Int, nClients: Int) -abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[CoherenceAgentRole])(implicit conf: LogicalNetworkConfiguration) extends Component { +abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[CoherenceAgentRole])(implicit conf: LogicalNetworkConfiguration) extends Module { override val io: Vec[TileLinkType] val physicalNetworks: Seq[PhysicalNetwork] require(endpoints.length == conf.nEndpoints) } class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val src = UFix(width = conf.idBits) - val dst = UFix(width = conf.idBits) + val src = UInt(width = conf.idBits) + val dst = UInt(width = conf.idBits) } object FIFOedLogicalNetworkIOWrapper { - def apply[T <: Data](in: FIFOIO[T], src: UFix = UFix(0), dst: UFix = UFix(0))(implicit conf: LogicalNetworkConfiguration) = { - val shim = (new FIFOedLogicalNetworkIOWrapper(src, dst)){ in.bits.clone } + def apply[T <: Data](in: DecoupledIO[T], src: UInt = UInt(0), dst: UInt = UInt(0))(implicit conf: LogicalNetworkConfiguration) = { + val shim = Module((new FIFOedLogicalNetworkIOWrapper(src, dst)){ in.bits.clone }) shim.io.in.valid := in.valid shim.io.in.bits := in.bits in.ready := shim.io.in.ready shim.io.out } } -class FIFOedLogicalNetworkIOWrapper[T <: Data](src: UFix, dst: UFix)(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { +class FIFOedLogicalNetworkIOWrapper[T <: Data](src: UInt, dst: UInt)(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Module { val io = new Bundle { - val in = (new FIFOIO){ data }.flip - val out = (new FIFOIO){(new LogicalNetworkIO){ data }} + val in = Decoupled(data).flip + val out = Decoupled((new LogicalNetworkIO){data}) } io.out.valid := io.in.valid io.out.bits.payload := io.in.bits @@ -168,18 +168,18 @@ class FIFOedLogicalNetworkIOWrapper[T <: Data](src: UFix, dst: UFix)(data: => T) } object FIFOedLogicalNetworkIOUnwrapper { - def apply[T <: Data](in: FIFOIO[LogicalNetworkIO[T]])(implicit conf: LogicalNetworkConfiguration) = { - val shim = (new FIFOedLogicalNetworkIOUnwrapper){ in.bits.payload.clone } + def apply[T <: Data](in: DecoupledIO[LogicalNetworkIO[T]])(implicit conf: LogicalNetworkConfiguration) = { + val shim = Module((new FIFOedLogicalNetworkIOUnwrapper){ in.bits.payload.clone }) shim.io.in.valid := in.valid shim.io.in.bits := in.bits in.ready := shim.io.in.ready shim.io.out } } -class FIFOedLogicalNetworkIOUnwrapper[T <: Data]()(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { +class FIFOedLogicalNetworkIOUnwrapper[T <: Data]()(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Module { val io = new Bundle { - val in = (new FIFOIO){(new LogicalNetworkIO){ data }}.flip - val out = (new FIFOIO){ data } + val in = Decoupled((new LogicalNetworkIO){data}).flip + val out = Decoupled(data) } io.out.valid := io.in.valid io.out.bits := io.in.bits.payload diff --git a/uncore/src/slowio.scala b/uncore/src/slowio.scala index b56b1396..4d42fccf 100644 --- a/uncore/src/slowio.scala +++ b/uncore/src/slowio.scala @@ -1,71 +1,68 @@ package uncore import Chisel._ -class SlowIO[T <: Data](val divisor_max: Int)(data: => T) extends Component +class SlowIO[T <: Data](val divisor_max: Int)(data: => T) extends Module { val io = new Bundle { - val out_fast = new FIFOIO()(data).flip - val out_slow = new FIFOIO()(data) - - val in_fast = new FIFOIO()(data) - val in_slow = new FIFOIO()(data).flip - + val out_fast = Decoupled(data).flip + val out_slow = Decoupled(data) + val in_fast = Decoupled(data) + val in_slow = Decoupled(data).flip val clk_slow = Bool(OUTPUT) - - val set_divisor = new PipeIO()(Bits(width = 32)).flip + val set_divisor = Valid(Bits(width = 32)).flip val divisor = Bits(OUTPUT, 32) } require(divisor_max >= 8 && divisor_max <= 65536 && isPow2(divisor_max)) - val divisor = Reg(resetVal = UFix(divisor_max-1)) - val d_shadow = Reg(resetVal = UFix(divisor_max-1)) - val hold = Reg(resetVal = UFix(divisor_max/4-1)) - val h_shadow = Reg(resetVal = UFix(divisor_max/4-1)) + val divisor = RegReset(UInt(divisor_max-1)) + val d_shadow = RegReset(UInt(divisor_max-1)) + val hold = RegReset(UInt(divisor_max/4-1)) + val h_shadow = RegReset(UInt(divisor_max/4-1)) when (io.set_divisor.valid) { - d_shadow := io.set_divisor.bits(log2Up(divisor_max)-1, 0).toUFix - h_shadow := io.set_divisor.bits(log2Up(divisor_max)-1+16, 16).toUFix + d_shadow := io.set_divisor.bits(log2Up(divisor_max)-1, 0).toUInt + h_shadow := io.set_divisor.bits(log2Up(divisor_max)-1+16, 16).toUInt } - io.divisor := hold << UFix(16) | divisor + io.divisor := hold << UInt(16) | divisor - val count = Reg{UFix(width = log2Up(divisor_max))} - val clock = Reg{Bool()} - count := count + UFix(1) + val count = Reg{UInt(width = log2Up(divisor_max))} + val myclock = Reg{Bool()} + count := count + UInt(1) - val rising = count === (divisor >> UFix(1)) + val rising = count === (divisor >> UInt(1)) val falling = count === divisor - val held = count === (divisor >> UFix(1)) + hold + val held = count === (divisor >> UInt(1)) + hold when (falling) { divisor := d_shadow hold := h_shadow - count := UFix(0) - clock := Bool(false) + count := UInt(0) + myclock := Bool(false) } when (rising) { - clock := Bool(true) + myclock := Bool(true) } - val in_slow_rdy = Reg(resetVal = Bool(false)) - val out_slow_val = Reg(resetVal = Bool(false)) - val out_slow_bits = Reg() { data } + val in_slow_rdy = RegReset(Bool(false)) + val out_slow_val = RegReset(Bool(false)) + val out_slow_bits = Reg(data) - val fromhost_q = new Queue(1)(data) - fromhost_q.io.enq.valid := rising && (io.in_slow.valid && in_slow_rdy || reset) + val fromhost_q = Module(new Queue(data,1)) + fromhost_q.io.enq.valid := rising && (io.in_slow.valid && in_slow_rdy || this.getReset) fromhost_q.io.enq.bits := io.in_slow.bits fromhost_q.io.deq <> io.in_fast - val tohost_q = new Queue(1)(data) + val tohost_q = Module(new Queue(data,1)) tohost_q.io.enq <> io.out_fast tohost_q.io.deq.ready := rising && io.out_slow.ready && out_slow_val when (held) { in_slow_rdy := fromhost_q.io.enq.ready out_slow_val := tohost_q.io.deq.valid - out_slow_bits := Mux(reset, fromhost_q.io.deq.bits, tohost_q.io.deq.bits) + out_slow_bits := Mux(this.getReset, fromhost_q.io.deq.bits, tohost_q.io.deq.bits) } io.in_slow.ready := in_slow_rdy io.out_slow.valid := out_slow_val io.out_slow.bits := out_slow_bits - io.clk_slow := clock + io.clk_slow := myclock } diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index 6f7e6175..c16801ed 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -5,10 +5,11 @@ case class TileLinkConfiguration(co: CoherencePolicyWithUncached, ln: LogicalNet abstract trait TileLinkSubBundle extends Bundle { implicit val conf: TileLinkConfiguration + // TODO: override clone here, passing conf } trait HasPhysicalAddress extends TileLinkSubBundle { - val addr = UFix(width = PADDR_BITS - OFFSET_BITS) + val addr = UInt(width = PADDR_BITS - OFFSET_BITS) } trait HasClientTransactionId extends TileLinkSubBundle { @@ -29,7 +30,7 @@ trait MasterSourcedMessage extends SourcedMessage object Acquire { - def apply(a_type: Bits, addr: UFix, client_xact_id: UFix)(implicit conf: TileLinkConfiguration) = { + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { val acq = new Acquire acq.a_type := a_type acq.addr := addr @@ -39,7 +40,7 @@ object Acquire acq.atomic_opcode := Bits(0) acq } - def apply(a_type: Bits, addr: UFix, client_xact_id: UFix, write_mask: Bits)(implicit conf: TileLinkConfiguration) = { + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, write_mask: Bits)(implicit conf: TileLinkConfiguration) = { val acq = new Acquire acq.a_type := a_type acq.addr := addr @@ -49,7 +50,7 @@ object Acquire acq.atomic_opcode := Bits(0) acq } - def apply(a_type: Bits, addr: UFix, client_xact_id: UFix, subword_addr: UFix, atomic_opcode: UFix)(implicit conf: TileLinkConfiguration) = { + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt)(implicit conf: TileLinkConfiguration) = { val acq = new Acquire acq.a_type := a_type acq.addr := addr @@ -61,7 +62,7 @@ object Acquire } } class Acquire(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId { - val a_type = Bits(width = conf.co.acquireTypeBits) + val a_type = Bits(width = conf.co.acquireTypeWidth) val write_mask = Bits(width = ACQUIRE_WRITE_MASK_BITS) val subword_addr = Bits(width = ACQUIRE_SUBWORD_ADDR_BITS) val atomic_opcode = Bits(width = ACQUIRE_ATOMIC_OP_BITS) @@ -74,13 +75,13 @@ class AcquireData(implicit val conf: TileLinkConfiguration) extends ClientSource } class Probe(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage with HasPhysicalAddress with HasMasterTransactionId { - val p_type = Bits(width = conf.co.probeTypeBits) + val p_type = Bits(width = conf.co.probeTypeWidth) override def clone = { (new Probe).asInstanceOf[this.type] } } object Release { - def apply(r_type: Bits, addr: UFix, client_xact_id: UFix, master_xact_id: UFix)(implicit conf: TileLinkConfiguration) = { + def apply(r_type: Bits, addr: UInt, client_xact_id: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { val rel = new Release rel.r_type := r_type rel.addr := addr @@ -90,7 +91,7 @@ object Release } } class Release(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasMasterTransactionId { - val r_type = Bits(width = conf.co.releaseTypeBits) + val r_type = Bits(width = conf.co.releaseTypeWidth) override def clone = { (new Release).asInstanceOf[this.type] } } @@ -99,7 +100,7 @@ class ReleaseData(implicit val conf: TileLinkConfiguration) extends ClientSource } class Grant(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage with HasTileLinkData with HasClientTransactionId with HasMasterTransactionId { - val g_type = Bits(width = conf.co.grantTypeBits) + val g_type = Bits(width = conf.co.grantTypeWidth) override def clone = { (new Grant).asInstanceOf[this.type] } } @@ -111,13 +112,13 @@ class GrantAck(implicit val conf: TileLinkConfiguration) extends ClientSourcedMe trait DirectionalIO trait ClientSourcedIO extends DirectionalIO trait MasterSourcedIO extends DirectionalIO -class ClientSourcedFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) with ClientSourcedIO { +class ClientSourcedFIFOIO[T <: Data]()(data: => T) extends DecoupledIO(data) with ClientSourcedIO { override def clone = { new ClientSourcedFIFOIO()(data).asInstanceOf[this.type] } } class ClientSourcedDataIO[M <: Data, D <: Data]()(meta: => M, data: => D) extends PairedDataIO()(meta,data) with ClientSourcedIO { override def clone = { new ClientSourcedDataIO()(meta,data).asInstanceOf[this.type] } } -class MasterSourcedFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) with MasterSourcedIO { +class MasterSourcedFIFOIO[T <: Data]()(data: => T) extends DecoupledIO(data) with MasterSourcedIO { flip() override def clone = { new MasterSourcedFIFOIO()(data).asInstanceOf[this.type] } } @@ -142,25 +143,25 @@ class TileLinkIO(implicit conf: TileLinkConfiguration) extends UncachedTileLinkI /* * TODO: Merge the below classes into children of an abstract class in Chisel 2.0 -abstract class UncachedTileLinkIOArbiter(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Component { +abstract class UncachedTileLinkIOArbiter(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Module { def acquireClientXactId(in: Acquire, id: Int): Bits def grantClientXactId(in: Grant): Bits - def arbIdx(in: Grant): UFix + def arbIdx(in: Grant): UInt } */ -class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int)(implicit conf: TileLinkConfiguration) extends Component { +class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int)(implicit conf: TileLinkConfiguration) extends Module { implicit val (ln, co) = (conf.ln, conf.co) - def acquireClientXactId(in: Acquire, id: Int) = Cat(in.client_xact_id, UFix(id, log2Up(n))) - def grantClientXactId(in: Grant) = in.client_xact_id >> UFix(log2Up(n)) - def arbIdx(in: Grant) = in.client_xact_id(log2Up(n)-1,0).toUFix + def acquireClientXactId(in: Acquire, id: Int) = Cat(in.client_xact_id, UInt(id, log2Up(n))) + def grantClientXactId(in: Grant) = in.client_xact_id >> UInt(log2Up(n)) + def arbIdx(in: Grant) = in.client_xact_id(log2Up(n)-1,0).toUInt val io = new Bundle { - val in = Vec(n) { new UncachedTileLinkIO }.flip + val in = Vec.fill(n){new UncachedTileLinkIO}.flip val out = new UncachedTileLinkIO } def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) - val acq_arb = new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData}) + val acq_arb = Module(new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData})) io.out.acquire <> acq_arb.io.out io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { arb.data <> req.data @@ -170,14 +171,14 @@ class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int)(implicit conf: TileL req.meta.ready := arb.meta.ready }} - val grant_ack_arb = (new RRArbiter(n)){ (new LogicalNetworkIO){new GrantAck} } + val grant_ack_arb = Module(new RRArbiter((new LogicalNetworkIO){new GrantAck},n)) io.out.grant_ack <> grant_ack_arb.io.out grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } io.out.grant.ready := Bool(false) for (i <- 0 until n) { io.in(i).grant.valid := Bool(false) - when (arbIdx(io.out.grant.bits.payload) === UFix(i)) { + when (arbIdx(io.out.grant.bits.payload) === UInt(i)) { io.in(i).grant.valid := io.out.grant.valid io.out.grant.ready := io.in(i).grant.ready } @@ -186,18 +187,18 @@ class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int)(implicit conf: TileL } } -class UncachedTileLinkIOArbiterThatPassesId(n: Int)(implicit conf: TileLinkConfiguration) extends Component { +class UncachedTileLinkIOArbiterThatPassesId(n: Int)(implicit conf: TileLinkConfiguration) extends Module { implicit val (ln, co) = (conf.ln, conf.co) def acquireClientXactId(in: Acquire, id: Int) = in.client_xact_id def grantClientXactId(in: Grant) = in.client_xact_id - def arbIdx(in: Grant): UFix = in.client_xact_id + def arbIdx(in: Grant): UInt = in.client_xact_id val io = new Bundle { - val in = Vec(n) { new UncachedTileLinkIO }.flip + val in = Vec.fill(n){new UncachedTileLinkIO}.flip val out = new UncachedTileLinkIO } def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) - val acq_arb = new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData}) + val acq_arb = Module(new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData})) io.out.acquire <> acq_arb.io.out io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { arb.data <> req.data @@ -207,14 +208,14 @@ class UncachedTileLinkIOArbiterThatPassesId(n: Int)(implicit conf: TileLinkConfi req.meta.ready := arb.meta.ready }} - val grant_ack_arb = (new RRArbiter(n)){ (new LogicalNetworkIO){new GrantAck} } + val grant_ack_arb = Module(new RRArbiter((new LogicalNetworkIO){new GrantAck},n)) io.out.grant_ack <> grant_ack_arb.io.out grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } io.out.grant.ready := Bool(false) for (i <- 0 until n) { io.in(i).grant.valid := Bool(false) - when (arbIdx(io.out.grant.bits.payload) === UFix(i)) { + when (arbIdx(io.out.grant.bits.payload) === UInt(i)) { io.in(i).grant.valid := io.out.grant.valid io.out.grant.ready := io.in(i).grant.ready } @@ -223,18 +224,18 @@ class UncachedTileLinkIOArbiterThatPassesId(n: Int)(implicit conf: TileLinkConfi } } -class UncachedTileLinkIOArbiterThatUsesNewId(n: Int)(implicit conf: TileLinkConfiguration) extends Component { +class UncachedTileLinkIOArbiterThatUsesNewId(n: Int)(implicit conf: TileLinkConfiguration) extends Module { implicit val (ln, co) = (conf.ln, conf.co) - def acquireClientXactId(in: Acquire, id: Int) = UFix(id, log2Up(n)) - def grantClientXactId(in: Grant) = UFix(0) // DNC + def acquireClientXactId(in: Acquire, id: Int) = UInt(id, log2Up(n)) + def grantClientXactId(in: Grant) = UInt(0) // DNC def arbIdx(in: Grant) = in.client_xact_id val io = new Bundle { - val in = Vec(n) { new UncachedTileLinkIO }.flip + val in = Vec.fill(n){new UncachedTileLinkIO}.flip val out = new UncachedTileLinkIO } def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) - val acq_arb = new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData}) + val acq_arb = Module(new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData})) io.out.acquire <> acq_arb.io.out io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { arb.data <> req.data @@ -244,14 +245,14 @@ class UncachedTileLinkIOArbiterThatUsesNewId(n: Int)(implicit conf: TileLinkConf req.meta.ready := arb.meta.ready }} - val grant_ack_arb = (new RRArbiter(n)){ (new LogicalNetworkIO){new GrantAck} } + val grant_ack_arb = Module(new RRArbiter((new LogicalNetworkIO){new GrantAck},n)) io.out.grant_ack <> grant_ack_arb.io.out grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } io.out.grant.ready := Bool(false) for (i <- 0 until n) { io.in(i).grant.valid := Bool(false) - when (arbIdx(io.out.grant.bits.payload) === UFix(i)) { + when (arbIdx(io.out.grant.bits.payload) === UInt(i)) { io.in(i).grant.valid := io.out.grant.valid io.out.grant.ready := io.in(i).grant.ready } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 60e100e0..dde4190a 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -1,11 +1,11 @@ package uncore import Chisel._ -abstract class CoherenceAgent(implicit conf: TileLinkConfiguration) extends Component with MasterCoherenceAgent { +abstract class CoherenceAgent(implicit conf: TileLinkConfiguration) extends Module with MasterCoherenceAgent { val io = new Bundle { val client = (new TileLinkIO).flip val master = new UncachedTileLinkIO - val incoherent = Vec(conf.ln.nClients) { Bool() }.asInput + val incoherent = Vec.fill(conf.ln.nClients){Bool()}.asInput } } @@ -17,8 +17,8 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration // Create SHRs for outstanding transactions val nTrackers = conf.nReleaseTransactions + conf.nAcquireTransactions - val trackerList = (0 until conf.nReleaseTransactions).map(new VoluntaryReleaseTracker(_, bankId)) ++ - (conf.nReleaseTransactions until nTrackers).map(new AcquireTracker(_, bankId)) + val trackerList = (0 until conf.nReleaseTransactions).map(id => Module(new VoluntaryReleaseTracker(id, bankId))) ++ + (conf.nReleaseTransactions until nTrackers).map(id => Module(new AcquireTracker(id, bankId))) // Propagate incoherence flags trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) @@ -28,7 +28,7 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val block_acquires = any_acquire_conflict - val alloc_arb = (new Arbiter(trackerList.size)) { Bool() } + val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.client alloc_arb.io.in(i).valid := t.acquire.meta.ready @@ -43,7 +43,7 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration alloc_arb.io.out.ready := acquire.meta.valid && !block_acquires // Handle probe request generation - val probe_arb = (new Arbiter(trackerList.size)){(new LogicalNetworkIO){ new Probe }} + val probe_arb = Module(new Arbiter((new LogicalNetworkIO){ new Probe }, trackerList.size)) io.client.probe <> probe_arb.io.out probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.probe } @@ -52,21 +52,21 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration val voluntary = co.isVoluntary(release.meta.bits.payload) val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) val block_releases = Bool(false) - val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)){Bool()}.lastIndexWhere{b: Bool => b} - //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UFix(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response - val release_idx = Mux(voluntary, UFix(0), release.meta.bits.payload.master_xact_id) + val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b} + //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response + val release_idx = Mux(voluntary, UInt(0), release.meta.bits.payload.master_xact_id) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.client t.release.meta.bits := release.meta.bits - t.release.meta.valid := release.meta.valid && (release_idx === UFix(i)) && !block_releases + t.release.meta.valid := release.meta.valid && (release_idx === UInt(i)) && !block_releases t.release.data.bits := release.data.bits t.release.data.valid := release.data.valid } - release.meta.ready := Vec(trackerList.map(_.io.client.release.meta.ready)){Bool()}(release_idx) && !block_releases + release.meta.ready := Vec(trackerList.map(_.io.client.release.meta.ready)).read(release_idx) && !block_releases release.data.ready := trackerList.map(_.io.client.release.data.ready).reduce(_||_) // Reply to initial requestor - val grant_arb = (new Arbiter(trackerList.size)){(new LogicalNetworkIO){ new Grant }} + val grant_arb = Module(new Arbiter((new LogicalNetworkIO){ new Grant }, trackerList.size)) io.client.grant <> grant_arb.io.out grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.grant } @@ -77,13 +77,13 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration ack.ready := Bool(true) // Create an arbiter for the one memory port - val outer_arb = new UncachedTileLinkIOArbiterThatPassesId(trackerList.size) + val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size)) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.master } io.master <> outer_arb.io.out } -abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) extends Component with OuterRequestGenerator { +abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) extends Module with OuterRequestGenerator { implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val io = new Bundle { val client = (new TileLinkIO).flip @@ -95,13 +95,13 @@ abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) exten } class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { - val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(4){ UFix() } - val state = Reg(resetVal = s_idle) + val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(4){ UInt() } + val state = RegReset(s_idle) val xact = Reg{ new Release } - val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(ln.nClients))) - val release_data_needs_write = Reg(resetVal = Bool(false)) - val mem_cmd_sent = Reg(resetVal = Bool(false)) - val cmd_to_write = Acquire(co.getUncachedWriteAcquireType, xact.addr, UFix(trackerId)) + val init_client_id_ = RegReset(UInt(0, width = log2Up(ln.nClients))) + val release_data_needs_write = RegReset(Bool(false)) + val mem_cmd_sent = RegReset(Bool(false)) + val cmd_to_write = Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId)) io.has_acquire_conflict := Bool(false) io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.meta.bits.payload.addr) && (state != s_idle) @@ -110,23 +110,23 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Cohe io.master.acquire.meta.valid := Bool(false) io.master.acquire.meta.bits.payload := cmd_to_write //TODO io.master.acquire.bits.header.dst - io.master.acquire.meta.bits.header.src := UFix(bankId) + io.master.acquire.meta.bits.header.src := UInt(bankId) io.master.acquire.data.valid := Bool(false) - io.master.acquire.data.bits.payload.data := UFix(0) + io.master.acquire.data.bits.payload.data := UInt(0) //TODO io.master.acquire_data.bits.header.dst - io.master.acquire.data.bits.header.src := UFix(bankId) + io.master.acquire.data.bits.header.src := UInt(bankId) io.client.acquire.meta.ready := Bool(false) io.client.acquire.data.ready := Bool(false) io.client.probe.valid := Bool(false) io.client.release.meta.ready := Bool(false) io.client.release.data.ready := Bool(false) // DNC io.client.grant.valid := Bool(false) - io.client.grant.bits.payload.g_type := co.getGrantType(xact, UFix(0)) + io.client.grant.bits.payload.g_type := co.getGrantType(xact, UInt(0)) io.client.grant.bits.payload.client_xact_id := xact.client_xact_id - io.client.grant.bits.payload.master_xact_id := UFix(trackerId) - io.client.grant.bits.payload.data := UFix(0) + io.client.grant.bits.payload.master_xact_id := UInt(trackerId) + io.client.grant.bits.payload.data := UInt(0) io.client.grant.bits.header.dst := init_client_id_ - io.client.grant.bits.header.src := UFix(bankId) + io.client.grant.bits.header.src := UInt(bankId) io.client.grant_ack.valid := Bool(false) switch (state) { @@ -136,7 +136,7 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Cohe xact := io.client.release.meta.bits.payload init_client_id_ := io.client.release.meta.bits.header.src release_data_needs_write := co.messageHasData(io.client.release.meta.bits.payload) - mem_cnt := UFix(0) + mem_cnt := UInt(0) mem_cmd_sent := Bool(false) state := s_mem } @@ -159,56 +159,54 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Cohe } class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { - val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } - val state = Reg(resetVal = s_idle) + val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UInt() } + val state = RegReset(s_idle) val xact = Reg{ new Acquire } - val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(ln.nClients))) - val release_data_client_id = Reg(resetVal = UFix(0, width = log2Up(ln.nClients))) + val init_client_id_ = RegReset(UInt(0, width = log2Up(ln.nClients))) + val release_data_client_id = RegReset(UInt(0, width = log2Up(ln.nClients))) //TODO: Will need id reg for merged release xacts - val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(ln.nClients))) + val init_sharer_cnt_ = RegReset(UInt(0, width = log2Up(ln.nClients))) val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt_) - val release_count = if (ln.nClients == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(ln.nClients))) - val probe_flags = Reg(resetVal = Bits(0, width = ln.nClients)) + val release_count = if (ln.nClients == 1) UInt(0) else RegReset(UInt(0, width = log2Up(ln.nClients))) + val probe_flags = RegReset(Bits(0, width = ln.nClients)) val curr_p_id = PriorityEncoder(probe_flags) - val x_needs_read = Reg(resetVal = Bool(false)) - val acquire_data_needs_write = Reg(resetVal = Bool(false)) - val release_data_needs_write = Reg(resetVal = Bool(false)) - val cmd_to_write = Acquire(co.getUncachedWriteAcquireType, xact.addr, UFix(trackerId)) - val cmd_to_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UFix(trackerId)) - val a_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val r_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val x_needs_read = RegReset(Bool(false)) + val acquire_data_needs_write = RegReset(Bool(false)) + val release_data_needs_write = RegReset(Bool(false)) + val cmd_to_write = Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId)) + val cmd_to_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) + val a_w_mem_cmd_sent = RegReset(Bool(false)) + val r_w_mem_cmd_sent = RegReset(Bool(false)) val probe_initial_flags = Bits(width = ln.nClients) probe_initial_flags := Bits(0) if (ln.nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence val probe_self = Bool(true) //co.needsSelfProbe(io.client.acquire.bits.payload) - val myflag = Mux(probe_self, Bits(0), UFixToOH(io.client.acquire.meta.bits.header.src(log2Up(ln.nClients)-1,0))) + val myflag = Mux(probe_self, Bits(0), UIntToOH(io.client.acquire.meta.bits.header.src(log2Up(ln.nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, io.client.acquire.meta.bits.payload.addr) && (state != s_idle) io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.meta.bits.payload.addr) && (state != s_idle) io.master.acquire.meta.valid := Bool(false) - io.master.acquire.meta.bits.payload := Acquire(co.getUncachedReadAcquireType, xact.addr, UFix(trackerId)) - //TODO io.master.acquire.bits.header.dst - io.master.acquire.meta.bits.header.src := UFix(bankId) + io.master.acquire.meta.bits.payload := Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) + io.master.acquire.meta.bits.header.src := UInt(bankId) io.master.acquire.data.valid := Bool(false) - io.master.acquire.data.bits.payload.data := UFix(0) - //TODO io.master.acquire_data.bits.header.dst - io.master.acquire.data.bits.header := UFix(bankId) + io.master.acquire.data.bits.payload.data := UInt(0) + io.master.acquire.data.bits.header.src := UInt(bankId) io.client.probe.valid := Bool(false) - io.client.probe.bits.payload.p_type := co.getProbeType(xact.a_type, UFix(0)) - io.client.probe.bits.payload.master_xact_id := UFix(trackerId) + io.client.probe.bits.payload.p_type := co.getProbeType(xact.a_type, UInt(0)) + io.client.probe.bits.payload.master_xact_id := UInt(trackerId) io.client.probe.bits.payload.addr := xact.addr - io.client.probe.bits.header.dst := UFix(0) - io.client.probe.bits.header.src := UFix(bankId) + io.client.probe.bits.header.dst := UInt(0) + io.client.probe.bits.header.src := UInt(bankId) io.client.grant.bits.payload.data := io.master.grant.bits.payload.data io.client.grant.bits.payload.g_type := grant_type io.client.grant.bits.payload.client_xact_id := xact.client_xact_id - io.client.grant.bits.payload.master_xact_id := UFix(trackerId) + io.client.grant.bits.payload.master_xact_id := UInt(trackerId) io.client.grant.bits.header.dst := init_client_id_ - io.client.grant.bits.header.src := UFix(bankId) - io.client.grant.valid := (io.master.grant.valid && (UFix(trackerId) === io.master.grant.bits.payload.client_xact_id)) + io.client.grant.bits.header.src := UInt(bankId) + io.client.grant.valid := (io.master.grant.valid && (UInt(trackerId) === io.master.grant.bits.payload.client_xact_id)) io.client.acquire.meta.ready := Bool(false) io.client.acquire.data.ready := Bool(false) io.client.release.meta.ready := Bool(false) @@ -222,11 +220,11 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgen when( io.client.acquire.meta.valid ) { xact := io.client.acquire.meta.bits.payload init_client_id_ := io.client.acquire.meta.bits.header.src - init_sharer_cnt_ := UFix(ln.nClients) // TODO: Broadcast only + init_sharer_cnt_ := UInt(ln.nClients) // TODO: Broadcast only acquire_data_needs_write := co.messageHasData(io.client.acquire.meta.bits.payload) - x_needs_read := co.needsOuterRead(io.client.acquire.meta.bits.payload.a_type, UFix(0)) + x_needs_read := co.needsOuterRead(io.client.acquire.meta.bits.payload.a_type, UInt(0)) probe_flags := probe_initial_flags - mem_cnt := UFix(0) + mem_cnt := UInt(0) r_w_mem_cmd_sent := Bool(false) a_w_mem_cmd_sent := Bool(false) if(ln.nClients > 1) { @@ -241,12 +239,12 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgen io.client.probe.bits.header.dst := curr_p_id } when(io.client.probe.ready) { - probe_flags := probe_flags & ~(UFixToOH(curr_p_id)) + probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) } io.client.release.meta.ready := Bool(true) when(io.client.release.meta.valid) { - if(ln.nClients > 1) release_count := release_count - UFix(1) - when(release_count === UFix(1)) { + if(ln.nClients > 1) release_count := release_count - UInt(1) + when(release_count === UInt(1)) { state := s_mem } when( co.messageHasData(io.client.release.meta.bits.payload)) { @@ -281,7 +279,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgen } . elsewhen (x_needs_read) { doOuterReqRead(io.master.acquire, cmd_to_read, x_needs_read) } . otherwise { - state := Mux(co.needsAckReply(xact.a_type, UFix(0)), s_ack, + state := Mux(co.needsAckReply(xact.a_type, UInt(0)), s_ack, Mux(co.requiresAck(io.client.grant.bits.payload), s_busy, s_idle)) } } @@ -290,7 +288,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgen when(io.client.grant.ready) { state := Mux(co.requiresAck(io.client.grant.bits.payload), s_busy, s_idle) } } is(s_busy) { // Nothing left to do but wait for transaction to complete - when (io.client.grant_ack.valid && io.client.grant_ack.bits.payload.master_xact_id === UFix(trackerId)) { + when (io.client.grant_ack.valid && io.client.grant_ack.bits.payload.master_xact_id === UInt(trackerId)) { state := s_idle } } @@ -298,10 +296,10 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgen } abstract trait OuterRequestGenerator { - val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) - val mem_cnt_next = mem_cnt + UFix(1) + val mem_cnt = RegReset(UInt(0, width = log2Up(REFILL_CYCLES))) + val mem_cnt_next = mem_cnt + UInt(1) - def doOuterReqWrite[T <: HasTileLinkData](master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], client_data: FIFOIO[LogicalNetworkIO[T]], cmd: Acquire, trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UFix) { + def doOuterReqWrite[T <: HasTileLinkData](master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], client_data: DecoupledIO[LogicalNetworkIO[T]], cmd: Acquire, trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UInt) { val do_write = client_data.valid && (client_data.bits.header.src === desired_client_data_src_id) master_acq.meta.bits.payload := cmd master_acq.data.bits.payload := client_data.bits.payload @@ -315,7 +313,7 @@ abstract trait OuterRequestGenerator { when(master_acq.data.ready) { client_data.ready:= Bool(true) mem_cnt := mem_cnt_next - when(mem_cnt === UFix(REFILL_CYCLES-1)) { + when(mem_cnt === UInt(REFILL_CYCLES-1)) { trigger := Bool(false) } } From d5c9eb0f54e7e27f0d07126c79dbc6c1e701a6c5 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Mon, 12 Aug 2013 20:52:18 -0700 Subject: [PATCH 187/374] reset -> resetVal, getReset -> reset --- uncore/src/llc.scala | 2 +- uncore/src/slowio.scala | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index 3e7c2cd5..db49f027 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -370,7 +370,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[UInt], da val s3_rdy = Bool() val replay_s2_rdy = Bool() - val s1_valid = Reg(update = io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy, reset = Bool(false)) + val s1_valid = Reg(update = io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy, resetVal = Bool(false)) val s1 = Reg(new MemReqCmd) when (io.cpu.req_cmd.fire()) { s1 := io.cpu.req_cmd.bits } when (replay_s2 && replay_s2_rdy) { s1 := s2 } diff --git a/uncore/src/slowio.scala b/uncore/src/slowio.scala index 4d42fccf..b8be1ed7 100644 --- a/uncore/src/slowio.scala +++ b/uncore/src/slowio.scala @@ -47,7 +47,7 @@ class SlowIO[T <: Data](val divisor_max: Int)(data: => T) extends Module val out_slow_bits = Reg(data) val fromhost_q = Module(new Queue(data,1)) - fromhost_q.io.enq.valid := rising && (io.in_slow.valid && in_slow_rdy || this.getReset) + fromhost_q.io.enq.valid := rising && (io.in_slow.valid && in_slow_rdy || this.reset) fromhost_q.io.enq.bits := io.in_slow.bits fromhost_q.io.deq <> io.in_fast @@ -58,7 +58,7 @@ class SlowIO[T <: Data](val divisor_max: Int)(data: => T) extends Module when (held) { in_slow_rdy := fromhost_q.io.enq.ready out_slow_val := tohost_q.io.deq.valid - out_slow_bits := Mux(this.getReset, fromhost_q.io.deq.bits, tohost_q.io.deq.bits) + out_slow_bits := Mux(this.reset, fromhost_q.io.deq.bits, tohost_q.io.deq.bits) } io.in_slow.ready := in_slow_rdy From 9162fbc9b5dde864c85a7e0555eeccd05c84f866 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 12 Aug 2013 23:15:54 -0700 Subject: [PATCH 188/374] Clean up cloning in tilelink bundles --- uncore/src/tilelink.scala | 39 +++++++++------------------------------ 1 file changed, 9 insertions(+), 30 deletions(-) diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index c16801ed..fdd65094 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -5,7 +5,7 @@ case class TileLinkConfiguration(co: CoherencePolicyWithUncached, ln: LogicalNet abstract trait TileLinkSubBundle extends Bundle { implicit val conf: TileLinkConfiguration - // TODO: override clone here, passing conf + override def clone = this.getClass.getConstructors.head.newInstance(conf).asInstanceOf[this.type] } trait HasPhysicalAddress extends TileLinkSubBundle { @@ -30,7 +30,7 @@ trait MasterSourcedMessage extends SourcedMessage object Acquire { - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt)(implicit conf: TileLinkConfiguration): Acquire = { val acq = new Acquire acq.a_type := a_type acq.addr := addr @@ -40,24 +40,15 @@ object Acquire acq.atomic_opcode := Bits(0) acq } - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, write_mask: Bits)(implicit conf: TileLinkConfiguration) = { - val acq = new Acquire - acq.a_type := a_type - acq.addr := addr - acq.client_xact_id := client_xact_id + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, write_mask: Bits)(implicit conf: TileLinkConfiguration): Acquire = { + val acq = apply(a_type, addr, client_xact_id) acq.write_mask := write_mask - acq.subword_addr := Bits(0) - acq.atomic_opcode := Bits(0) acq } - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt)(implicit conf: TileLinkConfiguration) = { - val acq = new Acquire - acq.a_type := a_type - acq.addr := addr - acq.client_xact_id := client_xact_id + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt)(implicit conf: TileLinkConfiguration): Acquire = { + val acq = apply(a_type, addr, client_xact_id) acq.subword_addr := subword_addr acq.atomic_opcode := atomic_opcode - acq.write_mask := Bits(0) acq } } @@ -66,17 +57,12 @@ class Acquire(implicit val conf: TileLinkConfiguration) extends ClientSourcedMes val write_mask = Bits(width = ACQUIRE_WRITE_MASK_BITS) val subword_addr = Bits(width = ACQUIRE_SUBWORD_ADDR_BITS) val atomic_opcode = Bits(width = ACQUIRE_ATOMIC_OP_BITS) - override def clone = { (new Acquire).asInstanceOf[this.type] } } - -class AcquireData(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasTileLinkData { - override def clone = { (new AcquireData).asInstanceOf[this.type] } -} +class AcquireData(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasTileLinkData class Probe(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage with HasPhysicalAddress with HasMasterTransactionId { val p_type = Bits(width = conf.co.probeTypeWidth) - override def clone = { (new Probe).asInstanceOf[this.type] } } object Release @@ -92,22 +78,15 @@ object Release } class Release(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasMasterTransactionId { val r_type = Bits(width = conf.co.releaseTypeWidth) - override def clone = { (new Release).asInstanceOf[this.type] } } -class ReleaseData(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasTileLinkData { - override def clone = { (new ReleaseData).asInstanceOf[this.type] } -} +class ReleaseData(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasTileLinkData class Grant(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage with HasTileLinkData with HasClientTransactionId with HasMasterTransactionId { val g_type = Bits(width = conf.co.grantTypeWidth) - override def clone = { (new Grant).asInstanceOf[this.type] } -} - -class GrantAck(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasMasterTransactionId { - override def clone = { (new GrantAck).asInstanceOf[this.type] } } +class GrantAck(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasMasterTransactionId trait DirectionalIO trait ClientSourcedIO extends DirectionalIO From 7ff4126d049449aec3e4ff4e20fab8d2f08cd049 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 13 Aug 2013 00:01:11 -0700 Subject: [PATCH 189/374] Abstracted UncachedTileLinkIOArbiters --- uncore/src/tilelink.scala | 136 ++++++++++---------------------------- 1 file changed, 35 insertions(+), 101 deletions(-) diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index fdd65094..6a3c8e20 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -120,123 +120,57 @@ class TileLinkIO(implicit conf: TileLinkConfiguration) extends UncachedTileLinkI override def clone = { new TileLinkIO().asInstanceOf[this.type] } } -/* - * TODO: Merge the below classes into children of an abstract class in Chisel 2.0 -abstract class UncachedTileLinkIOArbiter(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Module { +abstract class UncachedTileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfiguration) extends Module { def acquireClientXactId(in: Acquire, id: Int): Bits def grantClientXactId(in: Grant): Bits def arbIdx(in: Grant): UInt -} -*/ -class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int)(implicit conf: TileLinkConfiguration) extends Module { implicit val (ln, co) = (conf.ln, conf.co) + val io = new Bundle { + val in = Vec.fill(n){new UncachedTileLinkIO}.flip + val out = new UncachedTileLinkIO + } + def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) + val acq_arb = Module(new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData})) + io.out.acquire <> acq_arb.io.out + io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { + arb.data <> req.data + arb.meta.valid := req.meta.valid + arb.meta.bits := req.meta.bits + arb.meta.bits.payload.client_xact_id := acquireClientXactId(req.meta.bits.payload, id) + req.meta.ready := arb.meta.ready + }} + + val grant_ack_arb = Module(new RRArbiter((new LogicalNetworkIO){new GrantAck},n)) + io.out.grant_ack <> grant_ack_arb.io.out + grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } + + io.out.grant.ready := Bool(false) + for (i <- 0 until n) { + io.in(i).grant.valid := Bool(false) + when (arbIdx(io.out.grant.bits.payload) === UInt(i)) { + io.in(i).grant.valid := io.out.grant.valid + io.out.grant.ready := io.in(i).grant.ready + } + io.in(i).grant.bits := io.out.grant.bits + io.in(i).grant.bits.payload.client_xact_id := grantClientXactId(io.out.grant.bits.payload) + } +} + +class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) { def acquireClientXactId(in: Acquire, id: Int) = Cat(in.client_xact_id, UInt(id, log2Up(n))) def grantClientXactId(in: Grant) = in.client_xact_id >> UInt(log2Up(n)) def arbIdx(in: Grant) = in.client_xact_id(log2Up(n)-1,0).toUInt - - val io = new Bundle { - val in = Vec.fill(n){new UncachedTileLinkIO}.flip - val out = new UncachedTileLinkIO - } - def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) - val acq_arb = Module(new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData})) - io.out.acquire <> acq_arb.io.out - io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { - arb.data <> req.data - arb.meta.valid := req.meta.valid - arb.meta.bits := req.meta.bits - arb.meta.bits.payload.client_xact_id := acquireClientXactId(req.meta.bits.payload, id) - req.meta.ready := arb.meta.ready - }} - - val grant_ack_arb = Module(new RRArbiter((new LogicalNetworkIO){new GrantAck},n)) - io.out.grant_ack <> grant_ack_arb.io.out - grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } - - io.out.grant.ready := Bool(false) - for (i <- 0 until n) { - io.in(i).grant.valid := Bool(false) - when (arbIdx(io.out.grant.bits.payload) === UInt(i)) { - io.in(i).grant.valid := io.out.grant.valid - io.out.grant.ready := io.in(i).grant.ready - } - io.in(i).grant.bits := io.out.grant.bits - io.in(i).grant.bits.payload.client_xact_id := grantClientXactId(io.out.grant.bits.payload) - } } -class UncachedTileLinkIOArbiterThatPassesId(n: Int)(implicit conf: TileLinkConfiguration) extends Module { - implicit val (ln, co) = (conf.ln, conf.co) +class UncachedTileLinkIOArbiterThatPassesId(n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) { def acquireClientXactId(in: Acquire, id: Int) = in.client_xact_id def grantClientXactId(in: Grant) = in.client_xact_id def arbIdx(in: Grant): UInt = in.client_xact_id - - val io = new Bundle { - val in = Vec.fill(n){new UncachedTileLinkIO}.flip - val out = new UncachedTileLinkIO - } - def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) - val acq_arb = Module(new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData})) - io.out.acquire <> acq_arb.io.out - io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { - arb.data <> req.data - arb.meta.valid := req.meta.valid - arb.meta.bits := req.meta.bits - arb.meta.bits.payload.client_xact_id := acquireClientXactId(req.meta.bits.payload, id) - req.meta.ready := arb.meta.ready - }} - - val grant_ack_arb = Module(new RRArbiter((new LogicalNetworkIO){new GrantAck},n)) - io.out.grant_ack <> grant_ack_arb.io.out - grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } - - io.out.grant.ready := Bool(false) - for (i <- 0 until n) { - io.in(i).grant.valid := Bool(false) - when (arbIdx(io.out.grant.bits.payload) === UInt(i)) { - io.in(i).grant.valid := io.out.grant.valid - io.out.grant.ready := io.in(i).grant.ready - } - io.in(i).grant.bits := io.out.grant.bits - io.in(i).grant.bits.payload.client_xact_id := grantClientXactId(io.out.grant.bits.payload) - } } -class UncachedTileLinkIOArbiterThatUsesNewId(n: Int)(implicit conf: TileLinkConfiguration) extends Module { - implicit val (ln, co) = (conf.ln, conf.co) +class UncachedTileLinkIOArbiterThatUsesNewId(n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) { def acquireClientXactId(in: Acquire, id: Int) = UInt(id, log2Up(n)) def grantClientXactId(in: Grant) = UInt(0) // DNC def arbIdx(in: Grant) = in.client_xact_id - - val io = new Bundle { - val in = Vec.fill(n){new UncachedTileLinkIO}.flip - val out = new UncachedTileLinkIO - } - def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) - val acq_arb = Module(new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData})) - io.out.acquire <> acq_arb.io.out - io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { - arb.data <> req.data - arb.meta.valid := req.meta.valid - arb.meta.bits := req.meta.bits - arb.meta.bits.payload.client_xact_id := acquireClientXactId(req.meta.bits.payload, id) - req.meta.ready := arb.meta.ready - }} - - val grant_ack_arb = Module(new RRArbiter((new LogicalNetworkIO){new GrantAck},n)) - io.out.grant_ack <> grant_ack_arb.io.out - grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } - - io.out.grant.ready := Bool(false) - for (i <- 0 until n) { - io.in(i).grant.valid := Bool(false) - when (arbIdx(io.out.grant.bits.payload) === UInt(i)) { - io.in(i).grant.valid := io.out.grant.valid - io.out.grant.ready := io.in(i).grant.ready - } - io.in(i).grant.bits := io.out.grant.bits - io.in(i).grant.bits.payload.client_xact_id := grantClientXactId(io.out.grant.bits.payload) - } } - From 1308c08baa0eccdd98c3ec022a9de6217cc11cf4 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 13 Aug 2013 17:52:53 -0700 Subject: [PATCH 190/374] Reg standardization --- uncore/src/llc.scala | 2 +- uncore/src/network.scala | 42 ++++++++++++---------------------------- uncore/src/slowio.scala | 4 ++-- 3 files changed, 15 insertions(+), 33 deletions(-) diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index 3e7c2cd5..cd4341bb 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -370,7 +370,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[UInt], da val s3_rdy = Bool() val replay_s2_rdy = Bool() - val s1_valid = Reg(update = io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy, reset = Bool(false)) + val s1_valid = Reg(updateData = io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy, resetData = Bool(false)) val s1 = Reg(new MemReqCmd) when (io.cpu.req_cmd.fire()) { s1 := io.cpu.req_cmd.bits } when (replay_s2 && replay_s2_rdy) { s1 := s2 } diff --git a/uncore/src/network.scala b/uncore/src/network.scala index 29c1dee8..8733e2a6 100644 --- a/uncore/src/network.scala +++ b/uncore/src/network.scala @@ -148,43 +148,25 @@ class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { object FIFOedLogicalNetworkIOWrapper { def apply[T <: Data](in: DecoupledIO[T], src: UInt = UInt(0), dst: UInt = UInt(0))(implicit conf: LogicalNetworkConfiguration) = { - val shim = Module((new FIFOedLogicalNetworkIOWrapper(src, dst)){ in.bits.clone }) - shim.io.in.valid := in.valid - shim.io.in.bits := in.bits - in.ready := shim.io.in.ready - shim.io.out + val out = Decoupled((new LogicalNetworkIO){in.bits.clone}).asDirectionless + out.valid := in.valid + out.bits.payload := in.bits + out.bits.header.dst := dst + out.bits.header.src := src + in.ready := out.ready + out } } -class FIFOedLogicalNetworkIOWrapper[T <: Data](src: UInt, dst: UInt)(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Module { - val io = new Bundle { - val in = Decoupled(data).flip - val out = Decoupled((new LogicalNetworkIO){data}) - } - io.out.valid := io.in.valid - io.out.bits.payload := io.in.bits - io.out.bits.header.dst := dst - io.out.bits.header.src := src - io.in.ready := io.out.ready -} object FIFOedLogicalNetworkIOUnwrapper { def apply[T <: Data](in: DecoupledIO[LogicalNetworkIO[T]])(implicit conf: LogicalNetworkConfiguration) = { - val shim = Module((new FIFOedLogicalNetworkIOUnwrapper){ in.bits.payload.clone }) - shim.io.in.valid := in.valid - shim.io.in.bits := in.bits - in.ready := shim.io.in.ready - shim.io.out + val out = Decoupled(in.bits.payload.clone).asDirectionless + out.valid := in.valid + out.bits := in.bits.payload + in.ready := out.ready + out } } -class FIFOedLogicalNetworkIOUnwrapper[T <: Data]()(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Module { - val io = new Bundle { - val in = Decoupled((new LogicalNetworkIO){data}).flip - val out = Decoupled(data) - } - io.out.valid := io.in.valid - io.out.bits := io.in.bits.payload - io.in.ready := io.out.ready -} class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends Bundle { val header = new LogicalHeader diff --git a/uncore/src/slowio.scala b/uncore/src/slowio.scala index 4d42fccf..93c1324a 100644 --- a/uncore/src/slowio.scala +++ b/uncore/src/slowio.scala @@ -47,7 +47,7 @@ class SlowIO[T <: Data](val divisor_max: Int)(data: => T) extends Module val out_slow_bits = Reg(data) val fromhost_q = Module(new Queue(data,1)) - fromhost_q.io.enq.valid := rising && (io.in_slow.valid && in_slow_rdy || this.getReset) + fromhost_q.io.enq.valid := rising && (io.in_slow.valid && in_slow_rdy || reset) fromhost_q.io.enq.bits := io.in_slow.bits fromhost_q.io.deq <> io.in_fast @@ -58,7 +58,7 @@ class SlowIO[T <: Data](val divisor_max: Int)(data: => T) extends Module when (held) { in_slow_rdy := fromhost_q.io.enq.ready out_slow_val := tohost_q.io.deq.valid - out_slow_bits := Mux(this.getReset, fromhost_q.io.deq.bits, tohost_q.io.deq.bits) + out_slow_bits := Mux(reset, fromhost_q.io.deq.bits, tohost_q.io.deq.bits) } io.in_slow.ready := in_slow_rdy From 17d404b325fa71f715f03356baa060386840a41d Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 15 Aug 2013 15:27:38 -0700 Subject: [PATCH 191/374] final Reg changes --- uncore/src/llc.scala | 28 ++++++++++++++-------------- uncore/src/memserdes.scala | 18 +++++++++--------- uncore/src/network.scala | 8 ++++---- uncore/src/slowio.scala | 12 ++++++------ uncore/src/uncore.scala | 32 ++++++++++++++++---------------- 5 files changed, 49 insertions(+), 49 deletions(-) diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index cd4341bb..f9907934 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -29,7 +29,7 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn val wdata = in.bits.wdata.toBits val wmask = in.bits.wmask.toBits val ren = in.valid && !in.bits.rw - val reg_ren = RegUpdate(ren) + val reg_ren = Reg(next=ren) val rdata = Vec.fill(nWide){Bits()} val r = Pipe(ren, in.bits.addr, postLatency) @@ -113,7 +113,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Module override def clone = new MSHR().asInstanceOf[this.type] } - val valid = Vec.fill(outstanding){RegReset(Bool(false))} + val valid = Vec.fill(outstanding){Reg(init=Bool(false))} val validBits = valid.toBits val freeId = PriorityEncoder(~validBits) val mshr = Vec.fill(outstanding){Reg(new MSHR)} @@ -191,12 +191,12 @@ class LLCWriteback(requestors: Int) extends Module val mem = new ioMemPipe } - val valid = RegReset(Bool(false)) + val valid = Reg(init=Bool(false)) val who = Reg(UInt()) val addr = Reg(UInt()) val cmd_sent = Reg(Bool()) val data_sent = Reg(Bool()) - val count = RegReset(UInt(0, log2Up(REFILL_CYCLES))) + val count = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) var anyReq = Bool(false) for (i <- 0 until requestors) { @@ -252,10 +252,10 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[UInt]) extends Modul } val q = Module(new Queue(new QEntry, latency+2)) val qReady = q.io.count <= UInt(q.entries-latency-1) - val valid = RegReset(Bool(false)) + val valid = Reg(init=Bool(false)) val req = Reg(io.req.bits.clone) - val count = RegReset(UInt(0, log2Up(REFILL_CYCLES))) - val refillCount = RegReset(UInt(0, log2Up(REFILL_CYCLES))) + val count = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) + val refillCount = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) when (data.io.in.valid && !io.mem_resp.valid) { count := count + UInt(1) @@ -307,7 +307,7 @@ class MemReqArb(n: Int) extends Module // UNTESTED val mem = new ioMem } - val lock = RegReset(Bool(false)) + val lock = Reg(init=Bool(false)) val locker = Reg(UInt()) val arb = Module(new RRArbiter(new MemReqCmd, n)) @@ -360,17 +360,17 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[UInt], da val data = Module(new LLCData(4, sets, ways, dataLeaf)) val writeback = Module(new LLCWriteback(2)) - val initCount = RegReset(UInt(0, log2Up(sets+1))) + val initCount = Reg(init=UInt(0, log2Up(sets+1))) val initialize = !initCount(log2Up(sets)) when (initialize) { initCount := initCount + UInt(1) } - val replay_s2 = RegReset(Bool(false)) - val s2_valid = RegReset(Bool(false)) + val replay_s2 = Reg(init=Bool(false)) + val s2_valid = Reg(init=Bool(false)) val s2 = Reg(new MemReqCmd) val s3_rdy = Bool() val replay_s2_rdy = Bool() - val s1_valid = Reg(updateData = io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy, resetData = Bool(false)) + val s1_valid = Reg(next = io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy, init = Bool(false)) val s1 = Reg(new MemReqCmd) when (io.cpu.req_cmd.fire()) { s1 := io.cpu.req_cmd.bits } when (replay_s2 && replay_s2_rdy) { s1 := s2 } @@ -451,7 +451,7 @@ class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module val do_enq = io.enq.fire() && !do_flow val do_deq = io.deq.fire() && !do_flow - val maybe_full = RegReset(Bool(false)) + val maybe_full = Reg(init=Bool(false)) val enq_ptr = Counter(do_enq, entries)._1 val deq_ptr = Counter(do_deq, entries)._1 when (do_enq != do_deq) { maybe_full := do_enq } @@ -509,7 +509,7 @@ class DRAMSideLLCNull(numRequests: Int, refillCycles: Int) extends Module val inc = Bool() val dec = Bool() - val count = RegReset(UInt(numEntries, size)) + val count = Reg(init=UInt(numEntries, size)) val watermark = count >= UInt(refillCycles) when (inc && !dec) { diff --git a/uncore/src/memserdes.scala b/uncore/src/memserdes.scala index 9badb5e9..7a021621 100644 --- a/uncore/src/memserdes.scala +++ b/uncore/src/memserdes.scala @@ -54,9 +54,9 @@ class MemSerdes(w: Int) extends Module val in_buf = Reg(Bits()) val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(5) { UInt() } - val state = RegReset(s_idle) - val send_cnt = RegReset(UInt(0, log2Up((max(abits, dbits)+w-1)/w))) - val data_send_cnt = RegReset(UInt(0, log2Up(REFILL_CYCLES))) + val state = Reg(init=s_idle) + val send_cnt = Reg(init=UInt(0, log2Up((max(abits, dbits)+w-1)/w))) + val data_send_cnt = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) val adone = io.narrow.req.ready && send_cnt === UInt((abits-1)/w) val ddone = io.narrow.req.ready && send_cnt === UInt((dbits-1)/w) @@ -96,9 +96,9 @@ class MemSerdes(w: Int) extends Module send_cnt := UInt(0) } - val recv_cnt = RegReset(UInt(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = RegReset(UInt(0, log2Up(REFILL_CYCLES))) - val resp_val = RegReset(Bool(false)) + val recv_cnt = Reg(init=UInt(0, log2Up((rbits+w-1)/w))) + val data_recv_cnt = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) + val resp_val = Reg(init=Bool(false)) resp_val := Bool(false) when (io.narrow.resp.valid) { @@ -129,14 +129,14 @@ class MemDesser(w: Int) extends Module // test rig side val rbits = io.wide.resp.bits.getWidth require(dbits >= abits && rbits >= dbits) - val recv_cnt = RegReset(UInt(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = RegReset(UInt(0, log2Up(REFILL_CYCLES))) + val recv_cnt = Reg(init=UInt(0, log2Up((rbits+w-1)/w))) + val data_recv_cnt = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) val adone = io.narrow.req.valid && recv_cnt === UInt((abits-1)/w) val ddone = io.narrow.req.valid && recv_cnt === UInt((dbits-1)/w) val rdone = io.narrow.resp.valid && recv_cnt === UInt((rbits-1)/w) val s_cmd_recv :: s_cmd :: s_data_recv :: s_data :: s_reply :: Nil = Enum(5) { UInt() } - val state = RegReset(s_cmd_recv) + val state = Reg(init=s_cmd_recv) val in_buf = Reg(Bits()) when (io.narrow.req.valid && io.narrow.req.ready || io.narrow.resp.valid) { diff --git a/uncore/src/network.scala b/uncore/src/network.scala index 8733e2a6..9de890b1 100644 --- a/uncore/src/network.scala +++ b/uncore/src/network.scala @@ -19,8 +19,8 @@ class PairedArbiterIO[M <: Data, D <: Data](n: Int)(m: => M, d: => D) extends Bu class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock: Option[M => Bool] = None)(meta: => M, data: => D) extends Module { require(isPow2(count)) val io = new PairedArbiterIO(n)(meta,data) - val locked = if(count > 1) RegReset(Bool(false)) else Bool(false) - val lockIdx = if(count > 1) RegReset(UInt(n-1)) else UInt(n-1) + val locked = if(count > 1) Reg(init=Bool(false)) else Bool(false) + val lockIdx = if(count > 1) Reg(init=UInt(n-1)) else UInt(n-1) val grant = List.fill(n)(Bool()) val meta_chosen = Bits(width = log2Up(n)) @@ -37,7 +37,7 @@ class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock io.data_chosen := Mux(locked, lockIdx, meta_chosen) if(count > 1){ - val cnt = RegReset(UInt(0, width = log2Up(count))) + val cnt = Reg(init=UInt(0, width = log2Up(count))) val cnt_next = cnt + UInt(1) when(io.out.data.fire()){ cnt := cnt_next @@ -54,7 +54,7 @@ class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock } } } - val last_grant = RegReset(Bits(0, log2Up(n))) + val last_grant = Reg(init=Bits(0, log2Up(n))) val ctrl = ArbiterCtrl((0 until n).map(i => io.in(i).meta.valid && UInt(i) > last_grant) ++ io.in.map(_.meta.valid)) (0 until n).map(i => grant(i) := ctrl(i) && UInt(i) > last_grant || ctrl(i + n)) diff --git a/uncore/src/slowio.scala b/uncore/src/slowio.scala index 93c1324a..79c837d1 100644 --- a/uncore/src/slowio.scala +++ b/uncore/src/slowio.scala @@ -14,10 +14,10 @@ class SlowIO[T <: Data](val divisor_max: Int)(data: => T) extends Module } require(divisor_max >= 8 && divisor_max <= 65536 && isPow2(divisor_max)) - val divisor = RegReset(UInt(divisor_max-1)) - val d_shadow = RegReset(UInt(divisor_max-1)) - val hold = RegReset(UInt(divisor_max/4-1)) - val h_shadow = RegReset(UInt(divisor_max/4-1)) + val divisor = Reg(init=UInt(divisor_max-1)) + val d_shadow = Reg(init=UInt(divisor_max-1)) + val hold = Reg(init=UInt(divisor_max/4-1)) + val h_shadow = Reg(init=UInt(divisor_max/4-1)) when (io.set_divisor.valid) { d_shadow := io.set_divisor.bits(log2Up(divisor_max)-1, 0).toUInt h_shadow := io.set_divisor.bits(log2Up(divisor_max)-1+16, 16).toUInt @@ -42,8 +42,8 @@ class SlowIO[T <: Data](val divisor_max: Int)(data: => T) extends Module myclock := Bool(true) } - val in_slow_rdy = RegReset(Bool(false)) - val out_slow_val = RegReset(Bool(false)) + val in_slow_rdy = Reg(init=Bool(false)) + val out_slow_val = Reg(init=Bool(false)) val out_slow_bits = Reg(data) val fromhost_q = Module(new Queue(data,1)) diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index dde4190a..bfc703fa 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -96,11 +96,11 @@ abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) exten class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(4){ UInt() } - val state = RegReset(s_idle) + val state = Reg(init=s_idle) val xact = Reg{ new Release } - val init_client_id_ = RegReset(UInt(0, width = log2Up(ln.nClients))) - val release_data_needs_write = RegReset(Bool(false)) - val mem_cmd_sent = RegReset(Bool(false)) + val init_client_id_ = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val release_data_needs_write = Reg(init=Bool(false)) + val mem_cmd_sent = Reg(init=Bool(false)) val cmd_to_write = Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId)) io.has_acquire_conflict := Bool(false) @@ -160,23 +160,23 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Cohe class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UInt() } - val state = RegReset(s_idle) + val state = Reg(init=s_idle) val xact = Reg{ new Acquire } - val init_client_id_ = RegReset(UInt(0, width = log2Up(ln.nClients))) - val release_data_client_id = RegReset(UInt(0, width = log2Up(ln.nClients))) + val init_client_id_ = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val release_data_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) //TODO: Will need id reg for merged release xacts - val init_sharer_cnt_ = RegReset(UInt(0, width = log2Up(ln.nClients))) + val init_sharer_cnt_ = Reg(init=UInt(0, width = log2Up(ln.nClients))) val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt_) - val release_count = if (ln.nClients == 1) UInt(0) else RegReset(UInt(0, width = log2Up(ln.nClients))) - val probe_flags = RegReset(Bits(0, width = ln.nClients)) + val release_count = if (ln.nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(ln.nClients))) + val probe_flags = Reg(init=Bits(0, width = ln.nClients)) val curr_p_id = PriorityEncoder(probe_flags) - val x_needs_read = RegReset(Bool(false)) - val acquire_data_needs_write = RegReset(Bool(false)) - val release_data_needs_write = RegReset(Bool(false)) + val x_needs_read = Reg(init=Bool(false)) + val acquire_data_needs_write = Reg(init=Bool(false)) + val release_data_needs_write = Reg(init=Bool(false)) val cmd_to_write = Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId)) val cmd_to_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) - val a_w_mem_cmd_sent = RegReset(Bool(false)) - val r_w_mem_cmd_sent = RegReset(Bool(false)) + val a_w_mem_cmd_sent = Reg(init=Bool(false)) + val r_w_mem_cmd_sent = Reg(init=Bool(false)) val probe_initial_flags = Bits(width = ln.nClients) probe_initial_flags := Bits(0) if (ln.nClients > 1) { @@ -296,7 +296,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgen } abstract trait OuterRequestGenerator { - val mem_cnt = RegReset(UInt(0, width = log2Up(REFILL_CYCLES))) + val mem_cnt = Reg(init=UInt(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UInt(1) def doOuterReqWrite[T <: HasTileLinkData](master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], client_data: DecoupledIO[LogicalNetworkIO[T]], cmd: Acquire, trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UInt) { From 3763cd00049bfad25d1857223af32df504042f77 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 15 Aug 2013 15:57:16 -0700 Subject: [PATCH 192/374] standardizing sbt build conventions --- uncore/README | 6 ++++++ uncore/build.sbt | 14 ++++++++++++++ uncore/src/{ => main/scala}/coherence.scala | 0 uncore/src/{ => main/scala}/consts.scala | 0 uncore/src/{ => main/scala}/llc.scala | 0 uncore/src/{ => main/scala}/memserdes.scala | 0 uncore/src/{ => main/scala}/network.scala | 0 uncore/src/{ => main/scala}/package.scala | 0 uncore/src/{ => main/scala}/slowio.scala | 0 uncore/src/{ => main/scala}/tilelink.scala | 0 uncore/src/{ => main/scala}/uncore.scala | 0 11 files changed, 20 insertions(+) create mode 100644 uncore/README create mode 100644 uncore/build.sbt rename uncore/src/{ => main/scala}/coherence.scala (100%) rename uncore/src/{ => main/scala}/consts.scala (100%) rename uncore/src/{ => main/scala}/llc.scala (100%) rename uncore/src/{ => main/scala}/memserdes.scala (100%) rename uncore/src/{ => main/scala}/network.scala (100%) rename uncore/src/{ => main/scala}/package.scala (100%) rename uncore/src/{ => main/scala}/slowio.scala (100%) rename uncore/src/{ => main/scala}/tilelink.scala (100%) rename uncore/src/{ => main/scala}/uncore.scala (100%) diff --git a/uncore/README b/uncore/README new file mode 100644 index 00000000..c920e19a --- /dev/null +++ b/uncore/README @@ -0,0 +1,6 @@ +This is the repository for uncore components assosciated with UCB_BAR chip +projects. To uses these modules, include this repo as a git submodule within +the your chip repository and add it as Project in your chip's build.scala. +These components are only dependent on Chisel, i.e. +lazy val uncore = Project("uncore", file("uncore"), settings = buildSettings) dependsOn(chisel) + diff --git a/uncore/build.sbt b/uncore/build.sbt new file mode 100644 index 00000000..d5f2e6ea --- /dev/null +++ b/uncore/build.sbt @@ -0,0 +1,14 @@ +organization := "edu.berkeley.cs" + +version := "2.0" + +name := "uncore" + +scalaVersion := "2.10.2" + +resolvers ++= Seq( + "Sonatype Snapshots" at "http://oss.sonatype.org/content/repositories/snapshots", + "Sonatype Releases" at "http://oss.sonatype.org/content/repositories/releases" +) + +libraryDependencies += "edu.berkeley.cs" %% "chisel" % "2.0" diff --git a/uncore/src/coherence.scala b/uncore/src/main/scala/coherence.scala similarity index 100% rename from uncore/src/coherence.scala rename to uncore/src/main/scala/coherence.scala diff --git a/uncore/src/consts.scala b/uncore/src/main/scala/consts.scala similarity index 100% rename from uncore/src/consts.scala rename to uncore/src/main/scala/consts.scala diff --git a/uncore/src/llc.scala b/uncore/src/main/scala/llc.scala similarity index 100% rename from uncore/src/llc.scala rename to uncore/src/main/scala/llc.scala diff --git a/uncore/src/memserdes.scala b/uncore/src/main/scala/memserdes.scala similarity index 100% rename from uncore/src/memserdes.scala rename to uncore/src/main/scala/memserdes.scala diff --git a/uncore/src/network.scala b/uncore/src/main/scala/network.scala similarity index 100% rename from uncore/src/network.scala rename to uncore/src/main/scala/network.scala diff --git a/uncore/src/package.scala b/uncore/src/main/scala/package.scala similarity index 100% rename from uncore/src/package.scala rename to uncore/src/main/scala/package.scala diff --git a/uncore/src/slowio.scala b/uncore/src/main/scala/slowio.scala similarity index 100% rename from uncore/src/slowio.scala rename to uncore/src/main/scala/slowio.scala diff --git a/uncore/src/tilelink.scala b/uncore/src/main/scala/tilelink.scala similarity index 100% rename from uncore/src/tilelink.scala rename to uncore/src/main/scala/tilelink.scala diff --git a/uncore/src/uncore.scala b/uncore/src/main/scala/uncore.scala similarity index 100% rename from uncore/src/uncore.scala rename to uncore/src/main/scala/uncore.scala From dc53529156d54eba953640e71f567c5bb9644cf4 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 21 Aug 2013 16:00:51 -0700 Subject: [PATCH 193/374] added resolver, bumped chisel dependency --- uncore/build.sbt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/uncore/build.sbt b/uncore/build.sbt index d5f2e6ea..abe95365 100644 --- a/uncore/build.sbt +++ b/uncore/build.sbt @@ -8,7 +8,9 @@ scalaVersion := "2.10.2" resolvers ++= Seq( "Sonatype Snapshots" at "http://oss.sonatype.org/content/repositories/snapshots", - "Sonatype Releases" at "http://oss.sonatype.org/content/repositories/releases" + "Sonatype Releases" at "http://oss.sonatype.org/content/repositories/releases", + "scct-github-repository" at "http://mtkopone.github.com/scct/maven-repo" + ) -libraryDependencies += "edu.berkeley.cs" %% "chisel" % "2.0" +libraryDependencies += "edu.berkeley.cs" %% "chisel" % "2.1-SNAPSHOT" From 9aff60f340bb3d64ed7a1bf482cf80b93714cb63 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 21 Aug 2013 16:16:42 -0700 Subject: [PATCH 194/374] whitespace error in build.sbt --- uncore/build.sbt | 1 - 1 file changed, 1 deletion(-) diff --git a/uncore/build.sbt b/uncore/build.sbt index abe95365..6ea793a4 100644 --- a/uncore/build.sbt +++ b/uncore/build.sbt @@ -10,7 +10,6 @@ resolvers ++= Seq( "Sonatype Snapshots" at "http://oss.sonatype.org/content/repositories/snapshots", "Sonatype Releases" at "http://oss.sonatype.org/content/repositories/releases", "scct-github-repository" at "http://mtkopone.github.com/scct/maven-repo" - ) libraryDependencies += "edu.berkeley.cs" %% "chisel" % "2.1-SNAPSHOT" From b01fe4f6aa80e7b452910e22bdbbb996a4d26cc5 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Sat, 24 Aug 2013 15:24:17 -0700 Subject: [PATCH 195/374] fix memserdes bit ordering --- uncore/src/memserdes.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/uncore/src/memserdes.scala b/uncore/src/memserdes.scala index b1272b28..2af04df8 100644 --- a/uncore/src/memserdes.scala +++ b/uncore/src/memserdes.scala @@ -107,8 +107,7 @@ class MemSerdes(w: Int) extends Component } io.wide.resp.valid := resp_val - io.wide.resp.bits.tag := in_buf(io.wide.resp.bits.tag.width-1,0) - io.wide.resp.bits.data := in_buf >> UFix(io.wide.resp.bits.tag.width) + io.wide.resp.bits := io.wide.resp.bits.fromBits(in_buf) } class MemDesserIO(w: Int) extends Bundle { From ee98cd8378c8b97e141f25390bb2f7cd1803e81d Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 10 Sep 2013 10:54:51 -0700 Subject: [PATCH 196/374] new enum syntax --- uncore/src/main/scala/coherence.scala | 66 +++++++++++++-------------- uncore/src/main/scala/memserdes.scala | 4 +- uncore/src/main/scala/uncore.scala | 4 +- 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 7109be94..dcb80c8e 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -96,10 +96,10 @@ class ThreeStateIncoherence extends IncoherentPolicy { def nProbeTypes = 0 def nReleaseTypes = 2 def nGrantTypes = 3 - val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(nClientStates){ UInt() } - val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(nAcquireTypes){ UInt() } - val releaseVoluntaryInvalidateData :: releaseInvalidateAck :: Nil = Enum(nReleaseTypes){ UInt() } - val grantVoluntaryAck :: grantData :: grantAck :: Nil = Enum(nGrantTypes){ UInt() } + val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(UInt(), nClientStates) + val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(UInt(), nAcquireTypes) + val releaseVoluntaryInvalidateData :: releaseInvalidateAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantVoluntaryAck :: grantData :: grantAck :: Nil = Enum(UInt(), nGrantTypes) val uncachedAcquireTypeList = List() val hasDataAcquireTypeList = List(acquireWriteback) val hasDataReleaseTypeList = List(acquireWriteback) @@ -156,13 +156,13 @@ class MICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 5 def nGrantTypes = 7 - val tileInvalid :: tileValid :: Nil = Enum(nClientStates){ UInt() } - val globalInvalid :: globalValid :: Nil = Enum(nMasterStates){ UInt() } + val tileInvalid :: tileValid :: Nil = Enum(UInt(), nClientStates) + val globalInvalid :: globalValid :: Nil = Enum(UInt(), nMasterStates) - val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UInt() } - val probeInvalidate :: probeCopy :: Nil = Enum(nProbeTypes){ UInt() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UInt() } - val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UInt() } + val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) + val probeInvalidate :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -298,13 +298,13 @@ class MEICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 7 def nGrantTypes = 8 - val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UInt() } - val globalInvalid :: globalExclusiveClean :: Nil = Enum(nMasterStates){ UInt() } + val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(UInt(), nClientStates) + val globalInvalid :: globalExclusiveClean :: Nil = Enum(UInt(), nMasterStates) - val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UInt() } - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UInt() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UInt() } - val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UInt() } + val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -456,13 +456,13 @@ class MSICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 7 def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UInt() } - val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(nMasterStates){ UInt() } + val tileInvalid :: tileShared :: tileExclusiveDirty :: Nil = Enum(UInt(), nClientStates) + val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(UInt(), nMasterStates) - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UInt() } - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UInt() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UInt() } - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UInt() } + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -617,13 +617,13 @@ class MESICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 7 def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UInt() } - val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(nMasterStates){ UInt() } + val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(UInt(), nClientStates) + val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(UInt(), nMasterStates) - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UInt() } - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UInt() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UInt() } - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UInt() } + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -782,12 +782,12 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 11 def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: tileSharedByTwo :: tileMigratoryClean :: tileMigratoryDirty :: Nil = Enum(nClientStates){ UInt() } + val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: tileSharedByTwo :: tileMigratoryClean :: tileMigratoryDirty :: Nil = Enum(UInt(), nClientStates) - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(nAcquireTypes){ UInt() } - val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(nProbeTypes){ UInt() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(nReleaseTypes){ UInt() } - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(nGrantTypes){ UInt() } + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) + val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(UInt(), nProbeTypes) + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(UInt(), nReleaseTypes) + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(UInt(), nGrantTypes) val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 39c30e56..42611bc6 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -53,7 +53,7 @@ class MemSerdes(w: Int) extends Module val out_buf = Reg(Bits()) val in_buf = Reg(Bits()) - val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(5) { UInt() } + val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(UInt(), 5) val state = Reg(init=s_idle) val send_cnt = Reg(init=UInt(0, log2Up((max(abits, dbits)+w-1)/w))) val data_send_cnt = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) @@ -134,7 +134,7 @@ class MemDesser(w: Int) extends Module // test rig side val ddone = io.narrow.req.valid && recv_cnt === UInt((dbits-1)/w) val rdone = io.narrow.resp.valid && recv_cnt === UInt((rbits-1)/w) - val s_cmd_recv :: s_cmd :: s_data_recv :: s_data :: s_reply :: Nil = Enum(5) { UInt() } + val s_cmd_recv :: s_cmd :: s_data_recv :: s_data :: s_reply :: Nil = Enum(UInt(), 5) val state = Reg(init=s_cmd_recv) val in_buf = Reg(Bits()) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index bfc703fa..adf5f638 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -95,7 +95,7 @@ abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) exten } class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { - val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(4){ UInt() } + val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) val xact = Reg{ new Release } val init_client_id_ = Reg(init=UInt(0, width = log2Up(ln.nClients))) @@ -159,7 +159,7 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Cohe } class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { - val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UInt() } + val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(UInt(), 5) val state = Reg(init=s_idle) val xact = Reg{ new Acquire } val init_client_id_ = Reg(init=UInt(0, width = log2Up(ln.nClients))) From 1cac26fd76f7eea8668420d7d63903b42653ccb8 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 10 Sep 2013 16:15:41 -0700 Subject: [PATCH 197/374] NetworkIOs no longer use thunks --- uncore/src/main/scala/network.scala | 54 ++++++++++++++-------------- uncore/src/main/scala/tilelink.scala | 30 ++++++++-------- uncore/src/main/scala/uncore.scala | 4 +-- 3 files changed, 44 insertions(+), 44 deletions(-) diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index 9de890b1..4905ffde 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -2,23 +2,23 @@ package uncore import Chisel._ import scala.collection.mutable.Stack -class PairedDataIO[M <: Data, D <: Data]()(m: => M, d: => D) extends Bundle { - val meta = Decoupled(m) - val data = Decoupled(d) - override def clone = { new PairedDataIO()(m,d).asInstanceOf[this.type] } +class PairedDataIO[M <: Data, D <: Data](mType: M, dType: D) extends Bundle { + val meta = Decoupled(mType) + val data = Decoupled(dType) + override def clone = { new PairedDataIO(mType, dType).asInstanceOf[this.type] } } -class PairedArbiterIO[M <: Data, D <: Data](n: Int)(m: => M, d: => D) extends Bundle { - val in = Vec.fill(n){new PairedDataIO()(m,d)}.flip - val out = new PairedDataIO()(m,d) - val meta_chosen = Bits(OUTPUT, log2Up(n)) - val data_chosen = Bits(OUTPUT, log2Up(n)) - override def clone = { new PairedArbiterIO(n)(m,d).asInstanceOf[this.type] } +class PairedArbiterIO[M <: Data, D <: Data](mType: M, dType: D, n: Int) extends Bundle { + val in = Vec.fill(n){new PairedDataIO(mType, dType)}.flip + val out = new PairedDataIO(mType, dType) + val meta_chosen = UInt(OUTPUT, log2Up(n)) + val data_chosen = UInt(OUTPUT, log2Up(n)) + override def clone = { new PairedArbiterIO(mType, dType, n).asInstanceOf[this.type] } } -class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock: Option[M => Bool] = None)(meta: => M, data: => D) extends Module { +class PairedLockingRRArbiter[M <: Data, D <: Data](mType: M, dType: D, n: Int, count: Int, needsLock: Option[M => Bool] = None) extends Module { require(isPow2(count)) - val io = new PairedArbiterIO(n)(meta,data) + val io = new PairedArbiterIO(mType, dType, n) val locked = if(count > 1) Reg(init=Bool(false)) else Bool(false) val lockIdx = if(count > 1) Reg(init=UInt(n-1)) else UInt(n-1) val grant = List.fill(n)(Bool()) @@ -68,10 +68,10 @@ class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock when (io.out.meta.fire()) { last_grant := meta_chosen } } -class PairedCrossbar[M <: Data, D <: Data](count: Int, needsLock: Option[PhysicalNetworkIO[M] => Bool] = None)(meta: => M, data: => D)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { +class PairedCrossbar[M <: Data, D <: Data](mType: M, dType: D, count: Int, needsLock: Option[PhysicalNetworkIO[M] => Bool] = None)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { val io = new Bundle { - val in = Vec.fill(conf.nEndpoints){new PairedDataIO()(new PhysicalNetworkIO()(meta),new PhysicalNetworkIO()(data))}.flip - val out = Vec.fill(conf.nEndpoints){new PairedDataIO()(new PhysicalNetworkIO()(meta),new PhysicalNetworkIO()(data))} + val in = Vec.fill(conf.nEndpoints){new PairedDataIO(new PhysicalNetworkIO(mType),new PhysicalNetworkIO(dType))}.flip + val out = Vec.fill(conf.nEndpoints){new PairedDataIO(new PhysicalNetworkIO(mType),new PhysicalNetworkIO(dType))} } val metaRdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints){Bool()}) @@ -79,7 +79,7 @@ class PairedCrossbar[M <: Data, D <: Data](count: Int, needsLock: Option[Physica val rdyVecs = metaRdyVecs zip dataRdyVecs io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { - val rrarb = Module(new PairedLockingRRArbiter(conf.nEndpoints, count, needsLock)(io.in(0).meta.bits.clone, io.in(0).data.bits.clone)) + val rrarb = Module(new PairedLockingRRArbiter(io.in(0).meta.bits.clone, io.in(0).data.bits.clone, conf.nEndpoints, count, needsLock)) rrarb.io.in zip io.in zip rdys._1 zip rdys._2 map { case (((arb, in), meta_rdy), data_rdy) => { arb.meta.valid := in.meta.valid && (in.meta.bits.header.dst === UInt(i)) arb.meta.bits := in.meta.bits @@ -103,18 +103,18 @@ class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle val dst = UInt(width = conf.idBits) } -class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { - val header = (new PhysicalHeader) - val payload = data - override def clone = { new PhysicalNetworkIO()(data).asInstanceOf[this.type] } +class PhysicalNetworkIO[T <: Data](dType: T)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { + val header = new PhysicalHeader + val payload = dType.clone + override def clone = { new PhysicalNetworkIO(dType).asInstanceOf[this.type] } } abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Module -class BasicCrossbar[T <: Data](count: Int = 1)(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { +class BasicCrossbar[T <: Data](dType: T, count: Int = 1)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { val io = new Bundle { - val in = Vec.fill(conf.nEndpoints){Decoupled((new PhysicalNetworkIO){data})}.flip - val out = Vec.fill(conf.nEndpoints){Decoupled((new PhysicalNetworkIO){data})} + val in = Vec.fill(conf.nEndpoints){Decoupled(new PhysicalNetworkIO(dType))}.flip + val out = Vec.fill(conf.nEndpoints){Decoupled(new PhysicalNetworkIO(dType))} } val rdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints)(Bool())) @@ -148,7 +148,7 @@ class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { object FIFOedLogicalNetworkIOWrapper { def apply[T <: Data](in: DecoupledIO[T], src: UInt = UInt(0), dst: UInt = UInt(0))(implicit conf: LogicalNetworkConfiguration) = { - val out = Decoupled((new LogicalNetworkIO){in.bits.clone}).asDirectionless + val out = Decoupled(new LogicalNetworkIO(in.bits.clone)).asDirectionless out.valid := in.valid out.bits.payload := in.bits out.bits.header.dst := dst @@ -168,8 +168,8 @@ object FIFOedLogicalNetworkIOUnwrapper { } } -class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends Bundle { +class LogicalNetworkIO[T <: Data](dType: T)(implicit conf: LogicalNetworkConfiguration) extends Bundle { val header = new LogicalHeader - val payload = data - override def clone = { new LogicalNetworkIO()(data).asInstanceOf[this.type] } + val payload = dType.clone + override def clone = { new LogicalNetworkIO(dType).asInstanceOf[this.type] } } diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 6a3c8e20..425cbccd 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -91,32 +91,32 @@ class GrantAck(implicit val conf: TileLinkConfiguration) extends ClientSourcedMe trait DirectionalIO trait ClientSourcedIO extends DirectionalIO trait MasterSourcedIO extends DirectionalIO -class ClientSourcedFIFOIO[T <: Data]()(data: => T) extends DecoupledIO(data) with ClientSourcedIO { - override def clone = { new ClientSourcedFIFOIO()(data).asInstanceOf[this.type] } +class ClientSourcedFIFOIO[T <: Data](dType: T) extends DecoupledIO(dType) with ClientSourcedIO { + override def clone = { new ClientSourcedFIFOIO(dType).asInstanceOf[this.type] } } -class ClientSourcedDataIO[M <: Data, D <: Data]()(meta: => M, data: => D) extends PairedDataIO()(meta,data) with ClientSourcedIO { - override def clone = { new ClientSourcedDataIO()(meta,data).asInstanceOf[this.type] } +class ClientSourcedDataIO[M <: Data, D <: Data](mType: M, dType: D) extends PairedDataIO(mType, dType) with ClientSourcedIO { + override def clone = { new ClientSourcedDataIO(mType, dType).asInstanceOf[this.type] } } -class MasterSourcedFIFOIO[T <: Data]()(data: => T) extends DecoupledIO(data) with MasterSourcedIO { +class MasterSourcedFIFOIO[T <: Data](dType: T) extends DecoupledIO(dType) with MasterSourcedIO { flip() - override def clone = { new MasterSourcedFIFOIO()(data).asInstanceOf[this.type] } + override def clone = { new MasterSourcedFIFOIO(dType).asInstanceOf[this.type] } } -class MasterSourcedDataIO[M <: Data, D <: Data]()(meta: => M, data: => D) extends PairedDataIO()(meta,data) with MasterSourcedIO { +class MasterSourcedDataIO[M <: Data, D <: Data](mType: M, dType: D) extends PairedDataIO(mType, dType) with MasterSourcedIO { flip() - override def clone = { new MasterSourcedDataIO()(meta,data).asInstanceOf[this.type] } + override def clone = { new MasterSourcedDataIO(mType, dType).asInstanceOf[this.type] } } class UncachedTileLinkIO(implicit conf: TileLinkConfiguration) extends Bundle { implicit val ln = conf.ln - val acquire = new ClientSourcedDataIO()(new LogicalNetworkIO()(new Acquire), new LogicalNetworkIO()(new AcquireData)) - val grant = new MasterSourcedFIFOIO()(new LogicalNetworkIO()(new Grant)) - val grant_ack = new ClientSourcedFIFOIO()(new LogicalNetworkIO()(new GrantAck)) + val acquire = new ClientSourcedDataIO(new LogicalNetworkIO(new Acquire), new LogicalNetworkIO(new AcquireData)) + val grant = new MasterSourcedFIFOIO(new LogicalNetworkIO(new Grant)) + val grant_ack = new ClientSourcedFIFOIO(new LogicalNetworkIO(new GrantAck)) override def clone = { new UncachedTileLinkIO().asInstanceOf[this.type] } } class TileLinkIO(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIO()(conf) { - val probe = new MasterSourcedFIFOIO()(new LogicalNetworkIO()(new Probe)) - val release = new ClientSourcedDataIO()(new LogicalNetworkIO()(new Release), new LogicalNetworkIO()(new ReleaseData)) + val probe = new MasterSourcedFIFOIO(new LogicalNetworkIO(new Probe)) + val release = new ClientSourcedDataIO(new LogicalNetworkIO(new Release), new LogicalNetworkIO(new ReleaseData)) override def clone = { new TileLinkIO().asInstanceOf[this.type] } } @@ -131,7 +131,7 @@ abstract class UncachedTileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfigur val out = new UncachedTileLinkIO } def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) - val acq_arb = Module(new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData})) + val acq_arb = Module(new PairedLockingRRArbiter(new LogicalNetworkIO(new Acquire), new LogicalNetworkIO(new AcquireData), n, REFILL_CYCLES, acqHasData _)) io.out.acquire <> acq_arb.io.out io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { arb.data <> req.data @@ -141,7 +141,7 @@ abstract class UncachedTileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfigur req.meta.ready := arb.meta.ready }} - val grant_ack_arb = Module(new RRArbiter((new LogicalNetworkIO){new GrantAck},n)) + val grant_ack_arb = Module(new RRArbiter(new LogicalNetworkIO(new GrantAck), n)) io.out.grant_ack <> grant_ack_arb.io.out grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index adf5f638..a24f309d 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -43,7 +43,7 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration alloc_arb.io.out.ready := acquire.meta.valid && !block_acquires // Handle probe request generation - val probe_arb = Module(new Arbiter((new LogicalNetworkIO){ new Probe }, trackerList.size)) + val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) io.client.probe <> probe_arb.io.out probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.probe } @@ -66,7 +66,7 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration release.data.ready := trackerList.map(_.io.client.release.data.ready).reduce(_||_) // Reply to initial requestor - val grant_arb = Module(new Arbiter((new LogicalNetworkIO){ new Grant }, trackerList.size)) + val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) io.client.grant <> grant_arb.io.out grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.grant } From cc7783404df6dd87b7eaef3aa93b2b80906e8a13 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 12 Sep 2013 16:09:53 -0700 Subject: [PATCH 198/374] Add memory command M_XA_XOR --- uncore/src/main/scala/consts.scala | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index 15bfee4d..3a1a8482 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -21,13 +21,14 @@ trait MemoryOpConstants { val M_XWR = Bits("b00001"); // int store val M_PFR = Bits("b00010"); // prefetch with intent to read val M_PFW = Bits("b00011"); // prefetch with intent to write + val M_XA_SWAP = Bits("b00100"); val M_FENCE = Bits("b00101"); // memory fence val M_XLR = Bits("b00110"); val M_XSC = Bits("b00111"); val M_XA_ADD = Bits("b01000"); - val M_XA_SWAP = Bits("b01001"); - val M_XA_AND = Bits("b01010"); - val M_XA_OR = Bits("b01011"); + val M_XA_XOR = Bits("b01001"); + val M_XA_OR = Bits("b01010"); + val M_XA_AND = Bits("b01011"); val M_XA_MIN = Bits("b01100"); val M_XA_MAX = Bits("b01101"); val M_XA_MINU = Bits("b01110"); @@ -35,7 +36,7 @@ trait MemoryOpConstants { val M_INV = Bits("b10000"); // write back and invalidate line val M_CLN = Bits("b10001"); // write back line - def isAMO(cmd: Bits) = cmd(3) + def isAMO(cmd: Bits) = cmd(3) || cmd === M_XA_SWAP def isPrefetch(cmd: Bits) = cmd === M_PFR || cmd === M_PFW def isRead(cmd: Bits) = cmd === M_XRD || cmd === M_XLR || isAMO(cmd) def isWrite(cmd: Bits) = cmd === M_XWR || cmd === M_XSC || isAMO(cmd) From 9bf10ae5d22ebcc927703d43842097867add7ea1 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 19 Sep 2013 15:26:36 -0700 Subject: [PATCH 199/374] remove extraneous toBits (need new Chisel) --- uncore/src/main/scala/llc.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index f9907934..6627527a 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -466,7 +466,7 @@ class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module val ram_addr = Reg(Bits()) val ram_out_valid = Reg(Bool()) ram_out_valid := Bool(false) - when (do_enq) { ram(enq_ptr) := io.enq.bits.toBits } + when (do_enq) { ram(enq_ptr) := io.enq.bits } when (io.deq.ready && (atLeastTwo || !io.deq.valid && !empty)) { ram_out_valid := Bool(true) ram_addr := Mux(io.deq.valid, deq_ptr + UInt(1), deq_ptr) From cc3dc1bd0fbed6aad7f8ba9ade655f80f5e12369 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Thu, 19 Sep 2013 20:10:56 -0700 Subject: [PATCH 200/374] bug fix --- uncore/src/main/scala/llc.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index f9907934..6627527a 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -466,7 +466,7 @@ class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module val ram_addr = Reg(Bits()) val ram_out_valid = Reg(Bool()) ram_out_valid := Bool(false) - when (do_enq) { ram(enq_ptr) := io.enq.bits.toBits } + when (do_enq) { ram(enq_ptr) := io.enq.bits } when (io.deq.ready && (atLeastTwo || !io.deq.valid && !empty)) { ram_out_valid := Bool(true) ram_addr := Mux(io.deq.valid, deq_ptr + UInt(1), deq_ptr) From 20246b373ebfdb265c7bc095364384f1caee028b Mon Sep 17 00:00:00 2001 From: Stephen Twigg Date: Tue, 24 Sep 2013 16:02:00 -0700 Subject: [PATCH 201/374] Properly ignore target files --- uncore/.gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 uncore/.gitignore diff --git a/uncore/.gitignore b/uncore/.gitignore new file mode 100644 index 00000000..555feb41 --- /dev/null +++ b/uncore/.gitignore @@ -0,0 +1,2 @@ +target/ +project/target/ From 42693d43adc3dbbb65b1b0a591d3a20db904a804 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 26 Sep 2013 09:51:14 -0700 Subject: [PATCH 202/374] simplify build.sbt --- uncore/build.sbt | 8 -------- 1 file changed, 8 deletions(-) diff --git a/uncore/build.sbt b/uncore/build.sbt index 6ea793a4..8794797d 100644 --- a/uncore/build.sbt +++ b/uncore/build.sbt @@ -5,11 +5,3 @@ version := "2.0" name := "uncore" scalaVersion := "2.10.2" - -resolvers ++= Seq( - "Sonatype Snapshots" at "http://oss.sonatype.org/content/repositories/snapshots", - "Sonatype Releases" at "http://oss.sonatype.org/content/repositories/releases", - "scct-github-repository" at "http://mtkopone.github.com/scct/maven-repo" -) - -libraryDependencies += "edu.berkeley.cs" %% "chisel" % "2.1-SNAPSHOT" From f440df5338d3c1e7ba56fcd57ca8fa2ae98f5f99 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Mon, 28 Oct 2013 22:37:41 -0700 Subject: [PATCH 203/374] rename M_FENCE to M_NOP --- uncore/src/main/scala/consts.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index 3a1a8482..e9d282ae 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -22,7 +22,7 @@ trait MemoryOpConstants { val M_PFR = Bits("b00010"); // prefetch with intent to read val M_PFW = Bits("b00011"); // prefetch with intent to write val M_XA_SWAP = Bits("b00100"); - val M_FENCE = Bits("b00101"); // memory fence + val M_NOP = Bits("b00101"); val M_XLR = Bits("b00110"); val M_XSC = Bits("b00111"); val M_XA_ADD = Bits("b01000"); From c350cbd6eae0153ed511ceac6b45a9476d52a965 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Thu, 7 Nov 2013 13:19:04 -0800 Subject: [PATCH 204/374] move htif to uncore --- uncore/src/main/scala/htif.scala | 267 +++++++++++++++++++++++++++++++ 1 file changed, 267 insertions(+) create mode 100644 uncore/src/main/scala/htif.scala diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala new file mode 100644 index 00000000..38fce75e --- /dev/null +++ b/uncore/src/main/scala/htif.scala @@ -0,0 +1,267 @@ +package rocket + +import Chisel._ +import Node._ +import uncore._ + +class HostIO(val w: Int) extends Bundle +{ + val clk = Bool(OUTPUT) + val clk_edge = Bool(OUTPUT) + val in = Decoupled(Bits(width = w)).flip + val out = Decoupled(Bits(width = w)) + val debug_stats_pcr = Bool(OUTPUT) +} + +class PCRReq extends Bundle +{ + val rw = Bool() + val addr = Bits(width = 5) + val data = Bits(width = 64) +} + +class HTIFIO(ntiles: Int) extends Bundle +{ + val reset = Bool(INPUT) + val id = UInt(INPUT, log2Up(ntiles)) + val pcr_req = Decoupled(new PCRReq).flip + val pcr_rep = Decoupled(Bits(width = 64)) + val ipi_req = Decoupled(Bits(width = log2Up(ntiles))) + val ipi_rep = Decoupled(Bool()).flip + val debug_stats_pcr = Bool(OUTPUT) + // wired directly to stats register + // expected to be used to quickly indicate to testbench to do logging b/c in 'interesting' work +} + +class SCRIO(n: Int) extends Bundle +{ + val rdata = Vec.fill(n){Bits(INPUT, 64)} + val wen = Bool(OUTPUT) + val waddr = UInt(OUTPUT, log2Up(n)) + val wdata = Bits(OUTPUT, 64) +} + +class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfiguration) extends Module with ClientCoherenceAgent +{ + implicit val (ln, co) = (conf.ln, conf.co) + val nTiles = ln.nClients-1 // This HTIF is itself a TileLink client + val io = new Bundle { + val host = new HostIO(w) + val cpu = Vec.fill(nTiles){new HTIFIO(nTiles).flip} + val mem = new TileLinkIO + val scr = new SCRIO(nSCR) + } + + io.host.debug_stats_pcr := io.cpu.map(_.debug_stats_pcr).reduce(_||_) + // system is 'interesting' if any tile is 'interesting' + + val short_request_bits = 64 + val long_request_bits = 576 + require(short_request_bits % w == 0) + + val rx_count_w = 13 + log2Up(64) - log2Up(w) // data size field is 12 bits + val rx_count = Reg(init=UInt(0,rx_count_w)) + val rx_shifter = Reg(Bits(width = short_request_bits)) + val rx_shifter_in = Cat(io.host.in.bits, rx_shifter(short_request_bits-1,w)) + val next_cmd = rx_shifter_in(3,0) + val cmd = Reg(Bits()) + val size = Reg(Bits()) + val pos = Reg(Bits()) + val seqno = Reg(Bits()) + val addr = Reg(Bits()) + when (io.host.in.valid && io.host.in.ready) { + rx_shifter := rx_shifter_in + rx_count := rx_count + UInt(1) + when (rx_count === UInt(short_request_bits/w-1)) { + cmd := next_cmd + size := rx_shifter_in(15,4) + pos := rx_shifter_in(15,4+OFFSET_BITS-3) + seqno := rx_shifter_in(23,16) + addr := rx_shifter_in(63,24) + } + } + + val rx_word_count = (rx_count >> UInt(log2Up(short_request_bits/w))) + val rx_word_done = io.host.in.valid && rx_count(log2Up(short_request_bits/w)-1,0).andR + val packet_ram_depth = long_request_bits/short_request_bits-1 + val packet_ram = Vec.fill(packet_ram_depth){Reg(Bits(width = short_request_bits))} + when (rx_word_done && io.host.in.ready) { + packet_ram(rx_word_count(log2Up(packet_ram_depth)-1,0) - UInt(1)) := rx_shifter_in + } + + val cmd_readmem :: cmd_writemem :: cmd_readcr :: cmd_writecr :: cmd_ack :: cmd_nack :: Nil = Enum(UInt(), 6) + + val pcr_addr = addr(io.cpu(0).pcr_req.bits.addr.width-1, 0) + val pcr_coreid = addr(log2Up(nTiles)-1+20+1,20) + val pcr_wdata = packet_ram(0) + + val bad_mem_packet = size(OFFSET_BITS-1-3,0).orR || addr(OFFSET_BITS-1-3,0).orR + val nack = Mux(cmd === cmd_readmem || cmd === cmd_writemem, bad_mem_packet, + Mux(cmd === cmd_readcr || cmd === cmd_writecr, size != UInt(1), + Bool(true))) + + val tx_count = Reg(init=UInt(0, rx_count_w)) + val tx_subword_count = tx_count(log2Up(short_request_bits/w)-1,0) + val tx_word_count = tx_count(rx_count_w-1, log2Up(short_request_bits/w)) + val packet_ram_raddr = tx_word_count(log2Up(packet_ram_depth)-1,0) - UInt(1) + when (io.host.out.valid && io.host.out.ready) { + tx_count := tx_count + UInt(1) + } + + val rx_done = rx_word_done && Mux(rx_word_count === UInt(0), next_cmd != cmd_writemem && next_cmd != cmd_writecr, rx_word_count === size || rx_word_count(log2Up(packet_ram_depth)-1,0) === UInt(0)) + val tx_size = Mux(!nack && (cmd === cmd_readmem || cmd === cmd_readcr || cmd === cmd_writecr), size, UInt(0)) + val tx_done = io.host.out.ready && tx_subword_count.andR && (tx_word_count === tx_size || tx_word_count > UInt(0) && packet_ram_raddr.andR) + + val mem_acked = Reg(init=Bool(false)) + val mem_gxid = Reg(Bits()) + val mem_gsrc = Reg(UInt(width = conf.ln.idBits)) + val mem_needs_ack = Reg(Bool()) + when (io.mem.grant.valid) { + mem_acked := Bool(true) + mem_gxid := io.mem.grant.bits.payload.master_xact_id + mem_gsrc := io.mem.grant.bits.header.src + mem_needs_ack := conf.co.requiresAck(io.mem.grant.bits.payload) + } + io.mem.grant.ready := Bool(true) + + val state_rx :: state_pcr_req :: state_pcr_resp :: state_mem_req :: state_mem_wdata :: state_mem_wresp :: state_mem_rdata :: state_mem_finish :: state_tx :: Nil = Enum(UInt(), 9) + val state = Reg(init=state_rx) + + val rx_cmd = Mux(rx_word_count === UInt(0), next_cmd, cmd) + when (state === state_rx && rx_done) { + state := Mux(rx_cmd === cmd_readmem || rx_cmd === cmd_writemem, state_mem_req, + Mux(rx_cmd === cmd_readcr || rx_cmd === cmd_writecr, state_pcr_req, + state_tx)) + } + + val mem_cnt = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) + val x_init = Module(new Queue(new Acquire, 1)) + when (state === state_mem_req && x_init.io.enq.ready) { + state := Mux(cmd === cmd_writemem, state_mem_wdata, state_mem_rdata) + } + when (state === state_mem_wdata && io.mem.acquire.data.ready) { + when (mem_cnt.andR) { + state := state_mem_wresp + } + mem_cnt := mem_cnt + UInt(1) + } + when (state === state_mem_wresp) { + when (mem_acked) { + state := state_mem_finish + mem_acked := Bool(false) + } + } + when (state === state_mem_rdata) { + when (io.mem.grant.valid) { + when (mem_cnt.andR) { + state := state_mem_finish + } + mem_cnt := mem_cnt + UInt(1) + } + mem_acked := Bool(false) + } + when (state === state_mem_finish && io.mem.grant_ack.ready) { + state := Mux(cmd === cmd_readmem || pos === UInt(1), state_tx, state_rx) + pos := pos - UInt(1) + addr := addr + UInt(1 << OFFSET_BITS-3) + } + when (state === state_tx && tx_done) { + when (tx_word_count === tx_size) { + rx_count := UInt(0) + tx_count := UInt(0) + } + state := Mux(cmd === cmd_readmem && pos != UInt(0), state_mem_req, state_rx) + } + + var mem_req_data: Bits = null + for (i <- 0 until MEM_DATA_BITS/short_request_bits) { + val idx = Cat(mem_cnt, UInt(i, log2Up(MEM_DATA_BITS/short_request_bits))) + when (state === state_mem_rdata && io.mem.grant.valid) { + packet_ram(idx) := io.mem.grant.bits.payload.data((i+1)*short_request_bits-1, i*short_request_bits) + } + mem_req_data = Cat(packet_ram(idx), mem_req_data) + } + x_init.io.enq.valid := state === state_mem_req + val init_addr = addr.toUInt >> UInt(OFFSET_BITS-3) + x_init.io.enq.bits := Mux(cmd === cmd_writemem, + Acquire(co.getUncachedWriteAcquireType, init_addr, UInt(0)), + Acquire(co.getUncachedReadAcquireType, init_addr, UInt(0))) + io.mem.acquire.meta <> FIFOedLogicalNetworkIOWrapper(x_init.io.deq, UInt(conf.ln.nClients), UInt(0)) // By convention HTIF is the client with the largest id + io.mem.acquire.data.valid := state === state_mem_wdata + io.mem.acquire.data.bits.payload.data := mem_req_data + io.mem.grant_ack.valid := (state === state_mem_finish) && mem_needs_ack + io.mem.grant_ack.bits.payload.master_xact_id := mem_gxid + io.mem.grant_ack.bits.header.dst := mem_gsrc + io.mem.probe.ready := Bool(false) + io.mem.release.meta.valid := Bool(false) + io.mem.release.data.valid := Bool(false) + + val pcrReadData = Reg(Bits(width = io.cpu(0).pcr_rep.bits.getWidth)) + for (i <- 0 until nTiles) { + val my_reset = Reg(init=Bool(true)) + val my_ipi = Reg(init=Bool(false)) + + val cpu = io.cpu(i) + val me = pcr_coreid === UInt(i) + cpu.pcr_req.valid := state === state_pcr_req && me && pcr_addr != UInt(pcr_RESET) + cpu.pcr_req.bits.rw := cmd === cmd_writecr + cpu.pcr_req.bits.addr := pcr_addr + cpu.pcr_req.bits.data := pcr_wdata + cpu.reset := my_reset + + when (cpu.ipi_rep.ready) { + my_ipi := Bool(false) + } + cpu.ipi_rep.valid := my_ipi + cpu.ipi_req.ready := Bool(true) + for (j <- 0 until nTiles) { + when (io.cpu(j).ipi_req.valid && io.cpu(j).ipi_req.bits === UInt(i)) { + my_ipi := Bool(true) + } + } + + when (cpu.pcr_req.valid && cpu.pcr_req.ready) { + state := state_pcr_resp + } + when (state === state_pcr_req && me && pcr_addr === UInt(pcr_RESET)) { + when (cmd === cmd_writecr) { + my_reset := pcr_wdata(0) + } + pcrReadData := my_reset.toBits + state := state_tx + } + + cpu.pcr_rep.ready := Bool(true) + when (cpu.pcr_rep.valid) { + pcrReadData := cpu.pcr_rep.bits + state := state_tx + } + } + + val scr_addr = addr(log2Up(nSCR)-1, 0) + val scr_rdata = Vec.fill(io.scr.rdata.size){Bits(width = 64)} + for (i <- 0 until scr_rdata.size) + scr_rdata(i) := io.scr.rdata(i) + scr_rdata(0) := UInt(nTiles) + scr_rdata(1) := UInt(((REFILL_CYCLES*MEM_DATA_BITS/8) << x_init.io.enq.bits.addr.getWidth) >> 20) + + io.scr.wen := Bool(false) + io.scr.wdata := pcr_wdata + io.scr.waddr := scr_addr.toUInt + when (state === state_pcr_req && pcr_coreid === SInt(-1)) { + io.scr.wen := cmd === cmd_writecr + pcrReadData := scr_rdata(scr_addr) + state := state_tx + } + + val tx_cmd = Mux(nack, cmd_nack, cmd_ack) + val tx_cmd_ext = Cat(Bits(0, 4-tx_cmd.getWidth), tx_cmd) + val tx_header = Cat(addr, seqno, tx_size, tx_cmd_ext) + val tx_data = Mux(tx_word_count === UInt(0), tx_header, + Mux(cmd === cmd_readcr || cmd === cmd_writecr, pcrReadData, + packet_ram(packet_ram_raddr))) + + io.host.in.ready := state === state_rx + io.host.out.valid := state === state_tx + io.host.out.bits := tx_data >> Cat(tx_count(log2Up(short_request_bits/w)-1,0), Bits(0, log2Up(w))) +} From f13d76628b122d2dfc3950aa0761bcfd0f7394fc Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Thu, 7 Nov 2013 15:42:10 -0800 Subject: [PATCH 205/374] forgot to put htif into uncore package --- uncore/src/main/scala/htif.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 38fce75e..ec905012 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -1,4 +1,4 @@ -package rocket +package uncore import Chisel._ import Node._ From 056bb156cab81319750c7de53a55af57dfee7599 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Wed, 20 Nov 2013 16:43:55 -0800 Subject: [PATCH 206/374] make CacheConstants an object --- uncore/src/main/scala/consts.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index e9d282ae..bbe6cbd5 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -54,6 +54,7 @@ trait AddressConstants { val PERM_BITS = 6; } +object CacheConstants extends CacheConstants trait CacheConstants { val CACHE_DATA_SIZE_IN_BYTES = 1 << 6 val OFFSET_BITS = log2Up(CACHE_DATA_SIZE_IN_BYTES) From acc0d2b06c6007993552cf949fcd3af809b5ce54 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 25 Nov 2013 04:34:16 -0800 Subject: [PATCH 207/374] Only use LSBs for HTIF control regs For now, at least... --- uncore/src/main/scala/htif.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index ec905012..cd5c7534 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -196,6 +196,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati io.mem.release.meta.valid := Bool(false) io.mem.release.data.valid := Bool(false) + val pcr_reset = UInt(pcr_RESET)(pcr_addr.getWidth-1,0) val pcrReadData = Reg(Bits(width = io.cpu(0).pcr_rep.bits.getWidth)) for (i <- 0 until nTiles) { val my_reset = Reg(init=Bool(true)) @@ -203,7 +204,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati val cpu = io.cpu(i) val me = pcr_coreid === UInt(i) - cpu.pcr_req.valid := state === state_pcr_req && me && pcr_addr != UInt(pcr_RESET) + cpu.pcr_req.valid := state === state_pcr_req && me && pcr_addr != pcr_reset cpu.pcr_req.bits.rw := cmd === cmd_writecr cpu.pcr_req.bits.addr := pcr_addr cpu.pcr_req.bits.data := pcr_wdata @@ -223,7 +224,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati when (cpu.pcr_req.valid && cpu.pcr_req.ready) { state := state_pcr_resp } - when (state === state_pcr_req && me && pcr_addr === UInt(pcr_RESET)) { + when (state === state_pcr_req && me && pcr_addr === pcr_reset) { when (cmd === cmd_writecr) { my_reset := pcr_wdata(0) } From 4f1213cb8b638067702500c1d2c274595c690543 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 13 Jan 2014 21:45:14 -0800 Subject: [PATCH 208/374] Fix Scala integer overflow --- uncore/src/main/scala/htif.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index cd5c7534..fdd1162c 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -244,7 +244,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati for (i <- 0 until scr_rdata.size) scr_rdata(i) := io.scr.rdata(i) scr_rdata(0) := UInt(nTiles) - scr_rdata(1) := UInt(((REFILL_CYCLES*MEM_DATA_BITS/8) << x_init.io.enq.bits.addr.getWidth) >> 20) + scr_rdata(1) := UInt((BigInt(REFILL_CYCLES*MEM_DATA_BITS/8) << x_init.io.enq.bits.addr.getWidth) >> 20) io.scr.wen := Bool(false) io.scr.wdata := pcr_wdata From 3e634aef1d3afedacc4035248d81bc742377aff5 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 22 Jan 2014 18:20:36 -0800 Subject: [PATCH 209/374] Fix HTIF for cache line sizes other than 64 B --- uncore/src/main/scala/htif.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index fdd1162c..4e894af8 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -56,7 +56,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati // system is 'interesting' if any tile is 'interesting' val short_request_bits = 64 - val long_request_bits = 576 + val long_request_bits = short_request_bits + MEM_DATA_BITS*REFILL_CYCLES require(short_request_bits % w == 0) val rx_count_w = 13 + log2Up(64) - log2Up(w) // data size field is 12 bits From bbf801023039b406f5c6ad2ce952441fcf29d644 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 21 Jan 2014 12:20:55 -0800 Subject: [PATCH 210/374] cleanups supporting uncore hierarchy --- uncore/src/main/scala/coherence.scala | 244 ++++++++++++++------------ uncore/src/main/scala/consts.scala | 17 +- uncore/src/main/scala/htif.scala | 4 +- uncore/src/main/scala/network.scala | 25 +-- uncore/src/main/scala/tilelink.scala | 192 +++++++++++++------- uncore/src/main/scala/uncore.scala | 8 +- 6 files changed, 284 insertions(+), 206 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index dcb80c8e..e0db85f1 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -1,10 +1,6 @@ package uncore import Chisel._ -abstract trait CoherenceAgentRole -abstract trait ClientCoherenceAgent extends CoherenceAgentRole -abstract trait MasterCoherenceAgent extends CoherenceAgentRole - abstract class CoherencePolicy { def nClientStates: Int def nMasterStates: Int @@ -31,41 +27,42 @@ abstract class CoherencePolicy { def newStateOnWriteback(): UInt def newStateOnFlush(): UInt def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt - def newStateOnProbe(incoming: Probe, state: UInt): Bits + def newStateOnProbe(incoming: Probe, state: UInt): UInt def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt def getProbeType(a_type: UInt, global_state: UInt): UInt - def getReleaseTypeOnCacheControl(cmd: UInt): Bits - def getReleaseTypeOnVoluntaryWriteback(): Bits - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits - def getGrantType(a_type: UInt, count: UInt): Bits - def getGrantType(rel: Release, count: UInt): Bits + def getReleaseTypeOnCacheControl(cmd: UInt): UInt + def getReleaseTypeOnVoluntaryWriteback(): UInt + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt + def getGrantType(a_type: UInt, count: UInt): UInt + def getGrantType(rel: Release, count: UInt): UInt def messageHasData (rel: SourcedMessage): Bool def messageUpdatesDataArray (reply: Grant): Bool def messageIsUncached(acq: Acquire): Bool - def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool + def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool def isVoluntary(rel: Release): Bool def isVoluntary(gnt: Grant): Bool def needsOuterRead(a_type: UInt, global_state: UInt): Bool def needsOuterWrite(a_type: UInt, global_state: UInt): Bool - def needsAckReply(a_type: UInt, global_state: UInt): Bool - def needsSelfProbe(acq: Acquire): Bool - def requiresAck(grant: Grant): Bool - def requiresAck(release: Release): Bool + def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool + def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool + def requiresSelfProbe(a_type: UInt): Bool + def requiresAckForGrant(g_type: UInt): Bool + def requiresAckForRelease(r_type: UInt): Bool def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool - def uSIntListContains(list: List[UInt], elem: UInt): Bool = list.map(elem === _).reduceLeft(_||_) + def uIntListContains(list: List[UInt], elem: UInt): Bool = list.map(elem === _).reduceLeft(_||_) } trait UncachedTransactions { - def getUncachedReadAcquireType: Bits - def getUncachedWriteAcquireType: Bits - def getUncachedReadWordAcquireType: Bits - def getUncachedWriteWordAcquireType: Bits - def getUncachedAtomicAcquireType: Bits + def getUncachedReadAcquireType: UInt + def getUncachedWriteAcquireType: UInt + def getUncachedReadWordAcquireType: UInt + def getUncachedWriteWordAcquireType: UInt + def getUncachedAtomicAcquireType: UInt def isUncachedReadTransaction(acq: Acquire): Bool } @@ -73,18 +70,19 @@ abstract class CoherencePolicyWithUncached extends CoherencePolicy with Uncached abstract class IncoherentPolicy extends CoherencePolicy { // UNIMPLEMENTED - def newStateOnProbe(incoming: Probe, state: UInt): Bits = state - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = Bits(0) - def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = Bool(false) - def getGrantType(a_type: UInt, count: UInt): Bits = Bits(0) - def getGrantType(rel: Release, count: UInt): Bits = Bits(0) + def newStateOnProbe(incoming: Probe, state: UInt): UInt = state + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = UInt(0) + def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = Bool(false) + def getGrantType(a_type: UInt, count: UInt): UInt = UInt(0) + def getGrantType(rel: Release, count: UInt): UInt = UInt(0) def getProbeType(a_type: UInt, global_state: UInt): UInt = UInt(0) def needsOuterRead(a_type: UInt, global_state: UInt): Bool = Bool(false) def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = Bool(false) - def needsAckReply(a_type: UInt, global_state: UInt): Bool = Bool(false) - def needsSelfProbe(acq: Acquire) = Bool(false) - def requiresAck(grant: Grant) = Bool(true) - def requiresAck(release: Release) = Bool(false) + def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = Bool(false) + def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = Bool(false) + def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresAckForGrant(g_type: UInt) = Bool(true) + def requiresAckForRelease(r_type: UInt) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = Bool(false) } @@ -135,17 +133,17 @@ class ThreeStateIncoherence extends IncoherentPolicy { def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadDirty, outstanding.a_type) } - def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): Bits = releaseVoluntaryInvalidateData + def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): UInt = releaseVoluntaryInvalidateData def messageHasData( msg: SourcedMessage ) = msg match { - case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) + case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) case rel: Release => Bool(false) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant) = (reply.g_type === grantData) - def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) } class MICoherence extends CoherencePolicyWithUncached { @@ -202,7 +200,7 @@ class MICoherence extends CoherencePolicyWithUncached { grantAtomicUncached -> tileInvalid )) } - def newStateOnProbe(incoming: Probe, state: UInt): Bits = { + def newStateOnProbe(incoming: Probe, state: UInt): UInt = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeCopy -> state @@ -220,9 +218,9 @@ class MICoherence extends CoherencePolicyWithUncached { def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = acquireReadExclusive def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = acquireReadExclusive - def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = { + def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeCopy -> releaseCopyData @@ -235,19 +233,19 @@ class MICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uSIntListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uIntListContains(hasDataReleaseTypeList, rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) - def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) + def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a_type: UInt, count: UInt): Bits = { + def getGrantType(a_type: UInt, count: UInt): UInt = { MuxLookup(a_type, grantReadUncached, Array( acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, @@ -258,7 +256,7 @@ class MICoherence extends CoherencePolicyWithUncached { )) } - def getGrantType(rel: Release, count: UInt): Bits = { + def getGrantType(rel: Release, count: UInt): UInt = { MuxLookup(rel.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) @@ -281,12 +279,15 @@ class MICoherence extends CoherencePolicyWithUncached { def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def needsAckReply(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { + needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) + } + def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck - def requiresAck(release: Release) = Bool(false) - def needsSelfProbe(acq: Acquire) = Bool(false) + def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck + def requiresAckForRelease(r_type: UInt) = Bool(false) + def requiresSelfProbe(a_type: UInt) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -350,7 +351,7 @@ class MEICoherence extends CoherencePolicyWithUncached { grantAtomicUncached -> tileInvalid )) } - def newStateOnProbe(incoming: Probe, state: UInt): Bits = { + def newStateOnProbe(incoming: Probe, state: UInt): UInt = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeDowngrade -> tileExclusiveClean, @@ -373,9 +374,9 @@ class MEICoherence extends CoherencePolicyWithUncached { def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, outstanding.a_type) } - def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = { + def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -390,19 +391,19 @@ class MEICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uSIntListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uIntListContains(hasDataReleaseTypeList, rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) - def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) + def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a_type: UInt, count: UInt): Bits = { + def getGrantType(a_type: UInt, count: UInt): UInt = { MuxLookup(a_type, grantReadUncached, Array( acquireReadExclusiveClean -> grantReadExclusive, acquireReadExclusiveDirty -> grantReadExclusive, @@ -413,7 +414,7 @@ class MEICoherence extends CoherencePolicyWithUncached { acquireAtomicUncached -> grantAtomicUncached )) } - def getGrantType(rel: Release, count: UInt): Bits = { + def getGrantType(rel: Release, count: UInt): UInt = { MuxLookup(rel.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) @@ -438,12 +439,15 @@ class MEICoherence extends CoherencePolicyWithUncached { def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def needsAckReply(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { + needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) + } + def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck - def requiresAck(release: Release) = Bool(false) - def needsSelfProbe(acq: Acquire) = Bool(false) + def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck + def requiresAckForRelease(r_type: UInt) = Bool(false) + def requiresSelfProbe(a_type: UInt) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -514,7 +518,7 @@ class MSICoherence extends CoherencePolicyWithUncached { grantAtomicUncached -> tileInvalid )) } - def newStateOnProbe(incoming: Probe, state: UInt): Bits = { + def newStateOnProbe(incoming: Probe, state: UInt): UInt = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeDowngrade -> tileShared, @@ -537,9 +541,9 @@ class MSICoherence extends CoherencePolicyWithUncached { def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, outstanding.a_type) } - def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = { + def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -554,19 +558,19 @@ class MSICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uSIntListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uIntListContains(hasDataReleaseTypeList, rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) - def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) + def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a_type: UInt, count: UInt): Bits = { + def getGrantType(a_type: UInt, count: UInt): UInt = { MuxLookup(a_type, grantReadUncached, Array( acquireReadShared -> Mux(count > UInt(0), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, @@ -577,7 +581,7 @@ class MSICoherence extends CoherencePolicyWithUncached { acquireAtomicUncached -> grantAtomicUncached )) } - def getGrantType(rel: Release, count: UInt): Bits = { + def getGrantType(rel: Release, count: UInt): UInt = { MuxLookup(rel.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) @@ -599,12 +603,15 @@ class MSICoherence extends CoherencePolicyWithUncached { def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def needsAckReply(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { + needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) + } + def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck - def requiresAck(release: Release) = Bool(false) - def needsSelfProbe(acq: Acquire) = Bool(false) + def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck + def requiresAckForRelease(r_type: UInt) = Bool(false) + def requiresSelfProbe(a_type: UInt) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -675,7 +682,7 @@ class MESICoherence extends CoherencePolicyWithUncached { grantAtomicUncached -> tileInvalid )) } - def newStateOnProbe(incoming: Probe, state: UInt): Bits = { + def newStateOnProbe(incoming: Probe, state: UInt): UInt = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeDowngrade -> tileShared, @@ -698,9 +705,9 @@ class MESICoherence extends CoherencePolicyWithUncached { def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, outstanding.a_type) } - def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = { + def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -715,19 +722,19 @@ class MESICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uSIntListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uIntListContains(hasDataReleaseTypeList, rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) - def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) + def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a_type: UInt, count: UInt): Bits = { + def getGrantType(a_type: UInt, count: UInt): UInt = { MuxLookup(a_type, grantReadUncached, Array( acquireReadShared -> Mux(count > UInt(0), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, @@ -738,7 +745,7 @@ class MESICoherence extends CoherencePolicyWithUncached { acquireAtomicUncached -> grantAtomicUncached )) } - def getGrantType(rel: Release, count: UInt): Bits = { + def getGrantType(rel: Release, count: UInt): UInt = { MuxLookup(rel.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) @@ -763,13 +770,16 @@ class MESICoherence extends CoherencePolicyWithUncached { def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def needsAckReply(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { + needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) + } + def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck - def requiresAck(release: Release) = Bool(false) - def needsSelfProbe(acq: Acquire) = Bool(false) + def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck + def requiresAckForRelease(r_type: UInt) = Bool(false) + def requiresSelfProbe(a_type: UInt) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -795,7 +805,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) def isHit (cmd: UInt, state: UInt): Bool = { - Mux(isWriteIntent(cmd), uSIntListContains(List(tileExclusiveClean, tileExclusiveDirty, tileMigratoryClean, tileMigratoryDirty), state), (state != tileInvalid)) + Mux(isWriteIntent(cmd), uIntListContains(List(tileExclusiveClean, tileExclusiveDirty, tileMigratoryClean, tileMigratoryDirty), state), (state != tileInvalid)) } def isValid (state: UInt): Bool = { state != tileInvalid @@ -807,8 +817,8 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( - M_INV -> uSIntListContains(List(tileExclusiveDirty,tileMigratoryDirty),state), - M_CLN -> uSIntListContains(List(tileExclusiveDirty,tileMigratoryDirty),state) + M_INV -> uIntListContains(List(tileExclusiveDirty,tileMigratoryDirty),state), + M_CLN -> uIntListContains(List(tileExclusiveDirty,tileMigratoryDirty),state) )) } def needsWriteback (state: UInt): Bool = { @@ -846,7 +856,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { acquireReadShared -> tileMigratoryClean)) )) } - def newStateOnProbe(incoming: Probe, state: UInt): Bits = { + def newStateOnProbe(incoming: Probe, state: UInt): UInt = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeInvalidateOthers -> tileInvalid, @@ -875,11 +885,11 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), outstanding.a_type) } - def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = { + def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> Mux(uSIntListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), + probeInvalidate -> Mux(uIntListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), releaseInvalidateDataMigratory, releaseInvalidateData), probeDowngrade -> Mux(state === tileMigratoryDirty, releaseDowngradeDataMigratory, releaseDowngradeData), probeCopy -> releaseCopyData @@ -894,19 +904,19 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uSIntListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uIntListContains(hasDataReleaseTypeList, rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { - uSIntListContains(List(grantReadShared, grantReadExclusive, grantReadMigratory), reply.g_type) + uIntListContains(List(grantReadShared, grantReadExclusive, grantReadMigratory), reply.g_type) } - def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) - def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) + def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a_type: UInt, count: UInt): Bits = { + def getGrantType(a_type: UInt, count: UInt): UInt = { MuxLookup(a_type, grantReadUncached, Array( acquireReadShared -> Mux(count > UInt(0), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? acquireReadExclusive -> grantReadExclusive, @@ -918,7 +928,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI? )) } - def getGrantType(rel: Release, count: UInt): Bits = { + def getGrantType(rel: Release, count: UInt): UInt = { MuxLookup(rel.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) @@ -944,12 +954,16 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached || a_type === acquireAtomicUncached) } - def needsAckReply(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { + needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) + } + + def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached ||a_type === acquireInvalidateOthers) } - def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck - def requiresAck(release: Release) = Bool(false) - def needsSelfProbe(acq: Acquire) = Bool(false) + def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck + def requiresAckForRelease(r_type: UInt) = Bool(false) + def requiresSelfProbe(a_type: UInt) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index bbe6cbd5..d4f7130c 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -6,14 +6,15 @@ import scala.math.max object MemoryOpConstants extends MemoryOpConstants trait MemoryOpConstants { - val MT_X = Bits("b???", 3); - val MT_B = Bits("b000", 3); - val MT_H = Bits("b001", 3); - val MT_W = Bits("b010", 3); - val MT_D = Bits("b011", 3); - val MT_BU = Bits("b100", 3); - val MT_HU = Bits("b101", 3); - val MT_WU = Bits("b110", 3); + val MT_SZ = 3 + val MT_X = Bits("b???") + val MT_B = Bits("b000") + val MT_H = Bits("b001") + val MT_W = Bits("b010") + val MT_D = Bits("b011") + val MT_BU = Bits("b100") + val MT_HU = Bits("b101") + val MT_WU = Bits("b110") val M_SZ = 5 val M_X = Bits("b?????"); diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 4e894af8..608ac051 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -41,7 +41,7 @@ class SCRIO(n: Int) extends Bundle val wdata = Bits(OUTPUT, 64) } -class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfiguration) extends Module with ClientCoherenceAgent +class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfiguration) extends Module { implicit val (ln, co) = (conf.ln, conf.co) val nTiles = ln.nClients-1 // This HTIF is itself a TileLink client @@ -120,7 +120,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati mem_acked := Bool(true) mem_gxid := io.mem.grant.bits.payload.master_xact_id mem_gsrc := io.mem.grant.bits.header.src - mem_needs_ack := conf.co.requiresAck(io.mem.grant.bits.payload) + mem_needs_ack := conf.co.requiresAckForGrant(io.mem.grant.bits.payload.g_type) } io.mem.grant.ready := Bool(true) diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index 4905ffde..613a3b65 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -68,11 +68,13 @@ class PairedLockingRRArbiter[M <: Data, D <: Data](mType: M, dType: D, n: Int, c when (io.out.meta.fire()) { last_grant := meta_chosen } } -class PairedCrossbar[M <: Data, D <: Data](mType: M, dType: D, count: Int, needsLock: Option[PhysicalNetworkIO[M] => Bool] = None)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { - val io = new Bundle { +class PairedCrossbarIO[M <: Data, D <: Data](mType: M, dType: D)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { val in = Vec.fill(conf.nEndpoints){new PairedDataIO(new PhysicalNetworkIO(mType),new PhysicalNetworkIO(dType))}.flip val out = Vec.fill(conf.nEndpoints){new PairedDataIO(new PhysicalNetworkIO(mType),new PhysicalNetworkIO(dType))} - } +} + +class PairedCrossbar[M <: Data, D <: Data](mType: M, dType: D, count: Int, needsLock: Option[PhysicalNetworkIO[M] => Bool] = None)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { + val io = new PairedCrossbarIO(mType, dType) val metaRdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints){Bool()}) val dataRdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints){Bool()}) @@ -111,11 +113,12 @@ class PhysicalNetworkIO[T <: Data](dType: T)(implicit conf: PhysicalNetworkConfi abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Module -class BasicCrossbar[T <: Data](dType: T, count: Int = 1)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { - val io = new Bundle { +class BasicCrossbarIO[T <: Data](dType: T)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { val in = Vec.fill(conf.nEndpoints){Decoupled(new PhysicalNetworkIO(dType))}.flip val out = Vec.fill(conf.nEndpoints){Decoupled(new PhysicalNetworkIO(dType))} - } +} +class BasicCrossbar[T <: Data](dType: T, count: Int = 1)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { + val io = new BasicCrossbarIO(dType) val rdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints)(Bool())) @@ -133,14 +136,12 @@ class BasicCrossbar[T <: Data](dType: T, count: Int = 1)(implicit conf: Physical } } -case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nMasters: Int, nClients: Int) - -abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[CoherenceAgentRole])(implicit conf: LogicalNetworkConfiguration) extends Module { - override val io: Vec[TileLinkType] - val physicalNetworks: Seq[PhysicalNetwork] - require(endpoints.length == conf.nEndpoints) +case class LogicalNetworkConfiguration(idBits: Int, nMasters: Int, nClients: Int) { + val nEndpoints = nMasters + nClients } +abstract class LogicalNetwork[TileLinkType <: Bundle](implicit conf: LogicalNetworkConfiguration) extends Module + class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { val src = UInt(width = conf.idBits) val dst = UInt(width = conf.idBits) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 425cbccd..cd50213f 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -40,20 +40,25 @@ object Acquire acq.atomic_opcode := Bits(0) acq } - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, write_mask: Bits)(implicit conf: TileLinkConfiguration): Acquire = { + def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, write_mask: Bits)(implicit conf: TileLinkConfiguration): Acquire = { val acq = apply(a_type, addr, client_xact_id) acq.write_mask := write_mask acq } - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt)(implicit conf: TileLinkConfiguration): Acquire = { + def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt)(implicit conf: TileLinkConfiguration): Acquire = { val acq = apply(a_type, addr, client_xact_id) acq.subword_addr := subword_addr acq.atomic_opcode := atomic_opcode acq } + def apply(a: Acquire)(implicit conf: TileLinkConfiguration): Acquire = { + val acq = new Acquire + acq := a + acq + } } class Acquire(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId { - val a_type = Bits(width = conf.co.acquireTypeWidth) + val a_type = UInt(width = conf.co.acquireTypeWidth) val write_mask = Bits(width = ACQUIRE_WRITE_MASK_BITS) val subword_addr = Bits(width = ACQUIRE_SUBWORD_ADDR_BITS) val atomic_opcode = Bits(width = ACQUIRE_ATOMIC_OP_BITS) @@ -61,13 +66,29 @@ class Acquire(implicit val conf: TileLinkConfiguration) extends ClientSourcedMes class AcquireData(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasTileLinkData +object Probe +{ + def apply(p_type: UInt, addr: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { + val prb = new Probe + prb.p_type := p_type + prb.addr := addr + prb.master_xact_id := master_xact_id + prb + } +} class Probe(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage with HasPhysicalAddress with HasMasterTransactionId { - val p_type = Bits(width = conf.co.probeTypeWidth) + val p_type = UInt(width = conf.co.probeTypeWidth) } object Release { - def apply(r_type: Bits, addr: UInt, client_xact_id: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { + def apply(r_type: UInt, addr: UInt)(implicit conf: TileLinkConfiguration) = { + val rel = new Release + rel.r_type := r_type + rel.addr := addr + rel + } + def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { val rel = new Release rel.r_type := r_type rel.addr := addr @@ -77,100 +98,141 @@ object Release } } class Release(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasMasterTransactionId { - val r_type = Bits(width = conf.co.releaseTypeWidth) + val r_type = UInt(width = conf.co.releaseTypeWidth) } class ReleaseData(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasTileLinkData class Grant(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage with HasTileLinkData with HasClientTransactionId with HasMasterTransactionId { - val g_type = Bits(width = conf.co.grantTypeWidth) + val g_type = UInt(width = conf.co.grantTypeWidth) } class GrantAck(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasMasterTransactionId -trait DirectionalIO -trait ClientSourcedIO extends DirectionalIO -trait MasterSourcedIO extends DirectionalIO -class ClientSourcedFIFOIO[T <: Data](dType: T) extends DecoupledIO(dType) with ClientSourcedIO { - override def clone = { new ClientSourcedFIFOIO(dType).asInstanceOf[this.type] } -} -class ClientSourcedDataIO[M <: Data, D <: Data](mType: M, dType: D) extends PairedDataIO(mType, dType) with ClientSourcedIO { - override def clone = { new ClientSourcedDataIO(mType, dType).asInstanceOf[this.type] } -} -class MasterSourcedFIFOIO[T <: Data](dType: T) extends DecoupledIO(dType) with MasterSourcedIO { - flip() - override def clone = { new MasterSourcedFIFOIO(dType).asInstanceOf[this.type] } -} -class MasterSourcedDataIO[M <: Data, D <: Data](mType: M, dType: D) extends PairedDataIO(mType, dType) with MasterSourcedIO { - flip() - override def clone = { new MasterSourcedDataIO(mType, dType).asInstanceOf[this.type] } -} class UncachedTileLinkIO(implicit conf: TileLinkConfiguration) extends Bundle { implicit val ln = conf.ln - val acquire = new ClientSourcedDataIO(new LogicalNetworkIO(new Acquire), new LogicalNetworkIO(new AcquireData)) - val grant = new MasterSourcedFIFOIO(new LogicalNetworkIO(new Grant)) - val grant_ack = new ClientSourcedFIFOIO(new LogicalNetworkIO(new GrantAck)) + val acquire = new PairedDataIO(new LogicalNetworkIO(new Acquire), new LogicalNetworkIO(new AcquireData)) + val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip + val grant_ack = new DecoupledIO(new LogicalNetworkIO(new GrantAck)) override def clone = { new UncachedTileLinkIO().asInstanceOf[this.type] } } class TileLinkIO(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIO()(conf) { - val probe = new MasterSourcedFIFOIO(new LogicalNetworkIO(new Probe)) - val release = new ClientSourcedDataIO(new LogicalNetworkIO(new Release), new LogicalNetworkIO(new ReleaseData)) + val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip + val release = new PairedDataIO(new LogicalNetworkIO(new Release), new LogicalNetworkIO(new ReleaseData)) override def clone = { new TileLinkIO().asInstanceOf[this.type] } } -abstract class UncachedTileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfiguration) extends Module { - def acquireClientXactId(in: Acquire, id: Int): Bits - def grantClientXactId(in: Grant): Bits - def arbIdx(in: Grant): UInt - +abstract class TileLinkArbiterLike(val arbN: Int)(implicit conf: TileLinkConfiguration) extends Module { implicit val (ln, co) = (conf.ln, conf.co) + + type MasterSourcedWithId = MasterSourcedMessage with HasClientTransactionId + type ClientSourcedWithId = ClientSourcedMessage with HasClientTransactionId + type ClientSourcedWithData = ClientSourcedMessage with HasTileLinkData + + def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int): Bits + def masterSourcedClientXactId(in: MasterSourcedWithId): Bits + def arbIdx(in: MasterSourcedWithId): UInt + + def hookupClientSource[M <: ClientSourcedWithId, D <: ClientSourcedWithData] + (ins: Seq[PairedDataIO[LogicalNetworkIO[M],LogicalNetworkIO[D]]], + out: PairedDataIO[LogicalNetworkIO[M],LogicalNetworkIO[D]]) { + def hasData(m: LogicalNetworkIO[M]) = co.messageHasData(m.payload) + val arb = Module(new PairedLockingRRArbiter(out.meta.bits.clone, out.data.bits.clone, + arbN, REFILL_CYCLES, hasData _)) + out <> arb.io.out + ins.zipWithIndex.zip(arb.io.in).map{ case ((req,id), arb) => { + arb.data <> req.data + arb.meta.valid := req.meta.valid + arb.meta.bits := req.meta.bits + arb.meta.bits.payload := req.meta.bits.payload + arb.meta.bits.payload.client_xact_id := clientSourcedClientXactId(req.meta.bits.payload, id) + req.meta.ready := arb.meta.ready + }} + } + + def hookupMasterSource[M <: MasterSourcedWithId] + (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], + out: DecoupledIO[LogicalNetworkIO[M]]) { + out.ready := Bool(false) + for (i <- 0 until arbN) { + ins(i).valid := Bool(false) + when (arbIdx(out.bits.payload) === UInt(i)) { + ins(i).valid := out.valid + out.ready := ins(i).ready + } + ins(i).bits := out.bits + ins(i).bits.payload.client_xact_id := masterSourcedClientXactId(out.bits.payload) + } + } +} + +abstract class UncachedTileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkArbiterLike(n)(conf) { val io = new Bundle { val in = Vec.fill(n){new UncachedTileLinkIO}.flip val out = new UncachedTileLinkIO } - def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) - val acq_arb = Module(new PairedLockingRRArbiter(new LogicalNetworkIO(new Acquire), new LogicalNetworkIO(new AcquireData), n, REFILL_CYCLES, acqHasData _)) - io.out.acquire <> acq_arb.io.out - io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { - arb.data <> req.data - arb.meta.valid := req.meta.valid - arb.meta.bits := req.meta.bits - arb.meta.bits.payload.client_xact_id := acquireClientXactId(req.meta.bits.payload, id) - req.meta.ready := arb.meta.ready - }} + + hookupClientSource(io.in.map(_.acquire), io.out.acquire) + hookupMasterSource(io.in.map(_.grant), io.out.grant) val grant_ack_arb = Module(new RRArbiter(new LogicalNetworkIO(new GrantAck), n)) io.out.grant_ack <> grant_ack_arb.io.out grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } +} - io.out.grant.ready := Bool(false) - for (i <- 0 until n) { - io.in(i).grant.valid := Bool(false) - when (arbIdx(io.out.grant.bits.payload) === UInt(i)) { - io.in(i).grant.valid := io.out.grant.valid - io.out.grant.ready := io.in(i).grant.ready - } - io.in(i).grant.bits := io.out.grant.bits - io.in(i).grant.bits.payload.client_xact_id := grantClientXactId(io.out.grant.bits.payload) +abstract class TileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkArbiterLike(n)(conf) { + val io = new Bundle { + val in = Vec.fill(n){new TileLinkIO}.flip + val out = new TileLinkIO } + + hookupClientSource(io.in.map(_.acquire), io.out.acquire) + hookupClientSource(io.in.map(_.release), io.out.release) + hookupMasterSource(io.in.map(_.grant), io.out.grant) + + io.in.map{ _.probe.valid := io.out.probe.valid } + io.in.map{ _.probe.bits := io.out.probe.bits } + io.out.probe.ready := io.in.map(_.probe.ready).reduce(_||_) + + val grant_ack_arb = Module(new RRArbiter(new LogicalNetworkIO(new GrantAck), n)) + io.out.grant_ack <> grant_ack_arb.io.out + grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } } -class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) { - def acquireClientXactId(in: Acquire, id: Int) = Cat(in.client_xact_id, UInt(id, log2Up(n))) - def grantClientXactId(in: Grant) = in.client_xact_id >> UInt(log2Up(n)) - def arbIdx(in: Grant) = in.client_xact_id(log2Up(n)-1,0).toUInt +abstract trait AppendsArbiterId { + val arbN: Int + def clientSourcedClientXactId(in: ClientSourcedMessage with HasClientTransactionId, id: Int) = + Cat(in.client_xact_id, UInt(id, log2Up(arbN))) + def masterSourcedClientXactId(in: MasterSourcedMessage with HasClientTransactionId) = + in.client_xact_id >> UInt(log2Up(arbN)) + def arbIdx(in: MasterSourcedMessage with HasClientTransactionId) = + in.client_xact_id(log2Up(arbN)-1,0).toUInt } -class UncachedTileLinkIOArbiterThatPassesId(n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) { - def acquireClientXactId(in: Acquire, id: Int) = in.client_xact_id - def grantClientXactId(in: Grant) = in.client_xact_id - def arbIdx(in: Grant): UInt = in.client_xact_id +abstract trait PassesId { + def clientSourcedClientXactId(in: ClientSourcedMessage with HasClientTransactionId, id: Int) = + in.client_xact_id + def masterSourcedClientXactId(in: MasterSourcedMessage with HasClientTransactionId) = + in.client_xact_id + def arbIdx(in: MasterSourcedMessage with HasClientTransactionId) = + in.client_xact_id } -class UncachedTileLinkIOArbiterThatUsesNewId(n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) { - def acquireClientXactId(in: Acquire, id: Int) = UInt(id, log2Up(n)) - def grantClientXactId(in: Grant) = UInt(0) // DNC - def arbIdx(in: Grant) = in.client_xact_id +abstract trait UsesNewId { + val arbN: Int + def clientSourcedClientXactId(in: ClientSourcedMessage with HasClientTransactionId, id: Int) = + UInt(id, log2Up(arbN)) + def masterSourcedClientXactId(in: MasterSourcedMessage with HasClientTransactionId) = + UInt(0) + def arbIdx(in: MasterSourcedMessage with HasClientTransactionId) = + in.client_xact_id } + +class UncachedTileLinkIOArbiterThatAppendsArbiterId(val n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) with AppendsArbiterId +class UncachedTileLinkIOArbiterThatPassesId(val n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) with PassesId +class UncachedTileLinkIOArbiterThatUsesNewId(val n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) with UsesNewId +class TileLinkIOArbiterThatAppendsArbiterId(val n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkIOArbiter(n)(conf) with AppendsArbiterId +class TileLinkIOArbiterThatPassesId(val n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkIOArbiter(n)(conf) with PassesId +class TileLinkIOArbiterThatUsesNewId(val n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkIOArbiter(n)(conf) with UsesNewId diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index a24f309d..8b410639 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -1,7 +1,7 @@ package uncore import Chisel._ -abstract class CoherenceAgent(implicit conf: TileLinkConfiguration) extends Module with MasterCoherenceAgent { +abstract class CoherenceAgent(implicit conf: TileLinkConfiguration) extends Module { val io = new Bundle { val client = (new TileLinkIO).flip val master = new UncachedTileLinkIO @@ -279,13 +279,13 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgen } . elsewhen (x_needs_read) { doOuterReqRead(io.master.acquire, cmd_to_read, x_needs_read) } . otherwise { - state := Mux(co.needsAckReply(xact.a_type, UInt(0)), s_ack, - Mux(co.requiresAck(io.client.grant.bits.payload), s_busy, s_idle)) + state := Mux(co.requiresDatalessGrant(xact.a_type, UInt(0)), s_ack, + Mux(co.requiresAckForGrant(io.client.grant.bits.payload.g_type), s_busy, s_idle)) } } is(s_ack) { io.client.grant.valid := Bool(true) - when(io.client.grant.ready) { state := Mux(co.requiresAck(io.client.grant.bits.payload), s_busy, s_idle) } + when(io.client.grant.ready) { state := Mux(co.requiresAckForGrant(io.client.grant.bits.payload.g_type), s_busy, s_idle) } } is(s_busy) { // Nothing left to do but wait for transaction to complete when (io.client.grant_ack.valid && io.client.grant_ack.bits.payload.master_xact_id === UInt(trackerId)) { From 02dbd6b0aaaf0967ea26afbc2e4d32d12882d1ac Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 12 Feb 2014 18:39:40 -0800 Subject: [PATCH 211/374] Don't assign to your own inputs --- uncore/src/main/scala/uncore.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 8b410639..67dfde85 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -127,7 +127,6 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Cohe io.client.grant.bits.payload.data := UInt(0) io.client.grant.bits.header.dst := init_client_id_ io.client.grant.bits.header.src := UInt(bankId) - io.client.grant_ack.valid := Bool(false) switch (state) { is(s_idle) { @@ -212,7 +211,6 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgen io.client.release.meta.ready := Bool(false) io.client.release.data.ready := Bool(false) io.master.grant.ready := io.client.grant.ready - io.client.grant_ack.valid := Bool(false) switch (state) { is(s_idle) { From dec98eb047db3d5184961e7297b3728fe7be80a9 Mon Sep 17 00:00:00 2001 From: Jim Lawson Date: Tue, 18 Mar 2014 11:37:08 -0700 Subject: [PATCH 212/374] Update library dependencies (jenkins builds) Add chisel as an explicit library dependency --- uncore/build.sbt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uncore/build.sbt b/uncore/build.sbt index 8794797d..565e5fc7 100644 --- a/uncore/build.sbt +++ b/uncore/build.sbt @@ -5,3 +5,5 @@ version := "2.0" name := "uncore" scalaVersion := "2.10.2" + +libraryDependencies += "edu.berkeley.cs" %% "chisel" % "latest.release" From a228dbfa0d76237258026ff08f76375edef6524b Mon Sep 17 00:00:00 2001 From: Jim Lawson Date: Thu, 20 Mar 2014 10:40:05 -0700 Subject: [PATCH 213/374] Revert "Update library dependencies (jenkins builds)" This reverts commit e7a12143d0bfe8b3b4a4dcc78d119a89553ade8c. --- uncore/build.sbt | 2 -- 1 file changed, 2 deletions(-) diff --git a/uncore/build.sbt b/uncore/build.sbt index 565e5fc7..8794797d 100644 --- a/uncore/build.sbt +++ b/uncore/build.sbt @@ -5,5 +5,3 @@ version := "2.0" name := "uncore" scalaVersion := "2.10.2" - -libraryDependencies += "edu.berkeley.cs" %% "chisel" % "latest.release" From fbca7c6bb35914ac0c3f043090d4e8d407306ae2 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sat, 29 Mar 2014 10:53:49 -0700 Subject: [PATCH 214/374] refactor ioMem and associcated constants. merge Aqcuire and AcquireData --- uncore/src/main/scala/coherence.scala | 17 -- uncore/src/main/scala/consts.scala | 6 - uncore/src/main/scala/htif.scala | 56 +++-- uncore/src/main/scala/llc.scala | 72 +++--- uncore/src/main/scala/memserdes.scala | 141 +++++++++--- uncore/src/main/scala/network.scala | 113 +-------- uncore/src/main/scala/tilelink.scala | 89 ++++--- uncore/src/main/scala/uncore.scala | 320 +++++++++++--------------- 8 files changed, 384 insertions(+), 430 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index e0db85f1..c9d0f4eb 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -48,7 +48,6 @@ abstract class CoherencePolicy { def needsOuterRead(a_type: UInt, global_state: UInt): Bool def needsOuterWrite(a_type: UInt, global_state: UInt): Bool def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool - def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool def requiresSelfProbe(a_type: UInt): Bool def requiresAckForGrant(g_type: UInt): Bool def requiresAckForRelease(r_type: UInt): Bool @@ -79,7 +78,6 @@ abstract class IncoherentPolicy extends CoherencePolicy { def needsOuterRead(a_type: UInt, global_state: UInt): Bool = Bool(false) def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = Bool(false) def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = Bool(false) - def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = Bool(false) def requiresSelfProbe(a_type: UInt) = Bool(false) def requiresAckForGrant(g_type: UInt) = Bool(true) def requiresAckForRelease(r_type: UInt) = Bool(false) @@ -282,9 +280,6 @@ class MICoherence extends CoherencePolicyWithUncached { def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) } - def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { - (a_type === acquireWriteUncached) - } def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = Bool(false) @@ -442,9 +437,6 @@ class MEICoherence extends CoherencePolicyWithUncached { def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) } - def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { - (a_type === acquireWriteUncached) - } def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = Bool(false) @@ -606,9 +598,6 @@ class MSICoherence extends CoherencePolicyWithUncached { def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) } - def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { - (a_type === acquireWriteUncached) - } def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = Bool(false) @@ -773,9 +762,6 @@ class MESICoherence extends CoherencePolicyWithUncached { def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) } - def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { - (a_type === acquireWriteUncached) - } def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) @@ -958,9 +944,6 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) } - def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { - (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached ||a_type === acquireInvalidateOthers) - } def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = Bool(false) diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index d4f7130c..b0cdcdd6 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -70,9 +70,3 @@ trait TileLinkSizeConstants { trait MemoryInterfaceConstants extends CacheConstants with AddressConstants -{ - val MEM_TAG_BITS = 5 - val MEM_DATA_BITS = 128 - val REFILL_CYCLES = CACHE_DATA_SIZE_IN_BYTES*8/MEM_DATA_BITS - val MEM_ADDR_BITS = PADDR_BITS - OFFSET_BITS -} diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 608ac051..c4d0ffb0 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -56,7 +56,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati // system is 'interesting' if any tile is 'interesting' val short_request_bits = 64 - val long_request_bits = short_request_bits + MEM_DATA_BITS*REFILL_CYCLES + val long_request_bits = short_request_bits + conf.dataBits require(short_request_bits % w == 0) val rx_count_w = 13 + log2Up(64) - log2Up(w) // data size field is 12 bits @@ -124,26 +124,23 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati } io.mem.grant.ready := Bool(true) - val state_rx :: state_pcr_req :: state_pcr_resp :: state_mem_req :: state_mem_wdata :: state_mem_wresp :: state_mem_rdata :: state_mem_finish :: state_tx :: Nil = Enum(UInt(), 9) + val state_rx :: state_pcr_req :: state_pcr_resp :: state_mem_rreq :: state_mem_wreq :: state_mem_rresp :: state_mem_wresp :: state_mem_finish :: state_tx :: Nil = Enum(UInt(), 9) val state = Reg(init=state_rx) val rx_cmd = Mux(rx_word_count === UInt(0), next_cmd, cmd) when (state === state_rx && rx_done) { - state := Mux(rx_cmd === cmd_readmem || rx_cmd === cmd_writemem, state_mem_req, + state := Mux(rx_cmd === cmd_readmem, state_mem_rreq, + Mux(rx_cmd === cmd_writemem, state_mem_wreq, Mux(rx_cmd === cmd_readcr || rx_cmd === cmd_writecr, state_pcr_req, - state_tx)) + state_tx))) } - val mem_cnt = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) - val x_init = Module(new Queue(new Acquire, 1)) - when (state === state_mem_req && x_init.io.enq.ready) { - state := Mux(cmd === cmd_writemem, state_mem_wdata, state_mem_rdata) + val acq_q = Module(new Queue(new Acquire, 1)) + when (state === state_mem_wreq && acq_q.io.enq.ready) { + state := state_mem_wresp } - when (state === state_mem_wdata && io.mem.acquire.data.ready) { - when (mem_cnt.andR) { - state := state_mem_wresp - } - mem_cnt := mem_cnt + UInt(1) + when (state === state_mem_rreq && acq_q.io.enq.ready) { + state := state_mem_rresp } when (state === state_mem_wresp) { when (mem_acked) { @@ -151,12 +148,9 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati mem_acked := Bool(false) } } - when (state === state_mem_rdata) { + when (state === state_mem_rresp) { when (io.mem.grant.valid) { - when (mem_cnt.andR) { - state := state_mem_finish - } - mem_cnt := mem_cnt + UInt(1) + state := state_mem_finish } mem_acked := Bool(false) } @@ -170,31 +164,33 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati rx_count := UInt(0) tx_count := UInt(0) } - state := Mux(cmd === cmd_readmem && pos != UInt(0), state_mem_req, state_rx) + state := Mux(cmd === cmd_readmem && pos != UInt(0), state_mem_rreq, state_rx) } var mem_req_data: Bits = null - for (i <- 0 until MEM_DATA_BITS/short_request_bits) { - val idx = Cat(mem_cnt, UInt(i, log2Up(MEM_DATA_BITS/short_request_bits))) - when (state === state_mem_rdata && io.mem.grant.valid) { + for (i <- 0 until conf.dataBits/short_request_bits) { + val idx = UInt(i, log2Up(conf.dataBits/short_request_bits)) + when (state === state_mem_rresp && io.mem.grant.valid) { packet_ram(idx) := io.mem.grant.bits.payload.data((i+1)*short_request_bits-1, i*short_request_bits) } mem_req_data = Cat(packet_ram(idx), mem_req_data) } - x_init.io.enq.valid := state === state_mem_req + acq_q.io.enq.valid := state === state_mem_rreq || state === state_mem_wreq val init_addr = addr.toUInt >> UInt(OFFSET_BITS-3) - x_init.io.enq.bits := Mux(cmd === cmd_writemem, + acq_q.io.enq.bits := Mux(cmd === cmd_writemem, Acquire(co.getUncachedWriteAcquireType, init_addr, UInt(0)), Acquire(co.getUncachedReadAcquireType, init_addr, UInt(0))) - io.mem.acquire.meta <> FIFOedLogicalNetworkIOWrapper(x_init.io.deq, UInt(conf.ln.nClients), UInt(0)) // By convention HTIF is the client with the largest id - io.mem.acquire.data.valid := state === state_mem_wdata - io.mem.acquire.data.bits.payload.data := mem_req_data + io.mem.acquire.valid := acq_q.io.deq.valid + acq_q.io.deq.ready := io.mem.acquire.ready + io.mem.acquire.bits.payload := acq_q.io.deq.bits + io.mem.acquire.bits.payload.data := mem_req_data + io.mem.acquire.bits.header.src := UInt(conf.ln.nClients) // By convention HTIF is the client with the largest id + io.mem.acquire.bits.header.dst := UInt(0) // DNC; Overwritten outside module io.mem.grant_ack.valid := (state === state_mem_finish) && mem_needs_ack io.mem.grant_ack.bits.payload.master_xact_id := mem_gxid io.mem.grant_ack.bits.header.dst := mem_gsrc io.mem.probe.ready := Bool(false) - io.mem.release.meta.valid := Bool(false) - io.mem.release.data.valid := Bool(false) + io.mem.release.valid := Bool(false) val pcr_reset = UInt(pcr_RESET)(pcr_addr.getWidth-1,0) val pcrReadData = Reg(Bits(width = io.cpu(0).pcr_rep.bits.getWidth)) @@ -244,7 +240,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati for (i <- 0 until scr_rdata.size) scr_rdata(i) := io.scr.rdata(i) scr_rdata(0) := UInt(nTiles) - scr_rdata(1) := UInt((BigInt(REFILL_CYCLES*MEM_DATA_BITS/8) << x_init.io.enq.bits.addr.getWidth) >> 20) + scr_rdata(1) := UInt((BigInt(conf.dataBits/8) << acq_q.io.enq.bits.addr.getWidth) >> 20) io.scr.wen := Bool(false) io.scr.wdata := pcr_wdata diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index 6627527a..1bfa4da7 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -72,42 +72,42 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn io.rdata := Mux1H(rdataSel, rdataDeep) } -class LLCDataReq(ways: Int) extends MemReqCmd +class LLCDataReq(ways: Int)(implicit conf: MemoryIFConfiguration) extends MemReqCmd { val way = UInt(width = log2Up(ways)) val isWriteback = Bool() - override def clone = new LLCDataReq(ways).asInstanceOf[this.type] + override def clone = new LLCDataReq(ways)(conf).asInstanceOf[this.type] } -class LLCTagReq(ways: Int) extends HasMemAddr +class LLCTagReq(ways: Int)(implicit val conf: MemoryIFConfiguration) extends HasMemAddr { val way = UInt(width = log2Up(ways)) - override def clone = new LLCTagReq(ways).asInstanceOf[this.type] + override def clone = new LLCTagReq(ways)(conf).asInstanceOf[this.type] } -class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Module +class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int)(implicit conf: MemoryIFConfiguration) extends Module { val io = new Bundle { val cpu = Decoupled(new MemReqCmd).flip val repl_way = UInt(INPUT, log2Up(ways)) val repl_dirty = Bool(INPUT) - val repl_tag = UInt(INPUT, MEM_ADDR_BITS - log2Up(sets)) + val repl_tag = UInt(INPUT, conf.addrBits - log2Up(sets)) val data = Decoupled(new LLCDataReq(ways)) val tag = Decoupled(new LLCTagReq(ways)) - val mem = new ioMemPipe + val mem = new MemPipeIO val mem_resp_set = UInt(OUTPUT, log2Up(sets)) val mem_resp_way = UInt(OUTPUT, log2Up(ways)) } class MSHR extends Bundle { - val addr = UInt(width = PADDR_BITS - OFFSET_BITS) + val addr = UInt(width = conf.addrBits) val way = UInt(width = log2Up(ways)) val tag = io.cpu.bits.tag.clone val refilled = Bool() - val refillCount = UInt(width = log2Up(REFILL_CYCLES)) + val refillCount = UInt(width = log2Up(refill_cycles)) val requested = Bool() val old_dirty = Bool() - val old_tag = UInt(width = MEM_ADDR_BITS - log2Up(sets)) + val old_tag = UInt(width = conf.addrBits - log2Up(sets)) val wb_busy = Bool() override def clone = new MSHR().asInstanceOf[this.type] @@ -139,7 +139,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Module val refillCount = mshr(refillId).refillCount when (io.mem.resp.valid) { mshr(refillId).refillCount := refillCount + UInt(1) - when (refillCount === UInt(REFILL_CYCLES-1)) { mshr(refillId).refilled := Bool(true) } + when (refillCount === UInt(refill_cycles-1)) { mshr(refillId).refilled := Bool(true) } } val replays = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).refilled):_*) @@ -183,12 +183,12 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Module io.mem_resp_way := mshr(refillId).way } -class LLCWriteback(requestors: Int) extends Module +class LLCWriteback(requestors: Int, refill_cycles: Int)(implicit conf: MemoryIFConfiguration) extends Module { val io = new Bundle { - val req = Vec.fill(requestors){Decoupled(UInt(width = MEM_ADDR_BITS)).flip } + val req = Vec.fill(requestors){Decoupled(UInt(width = conf.addrBits)).flip } val data = Vec.fill(requestors){Decoupled(new MemData).flip } - val mem = new ioMemPipe + val mem = new MemPipeIO } val valid = Reg(init=Bool(false)) @@ -196,7 +196,7 @@ class LLCWriteback(requestors: Int) extends Module val addr = Reg(UInt()) val cmd_sent = Reg(Bool()) val data_sent = Reg(Bool()) - val count = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) + val count = Reg(init=UInt(0, log2Up(refill_cycles))) var anyReq = Bool(false) for (i <- 0 until requestors) { @@ -216,7 +216,7 @@ class LLCWriteback(requestors: Int) extends Module when (io.mem.req_data.valid && io.mem.req_data.ready) { count := count + UInt(1) - when (count === UInt(REFILL_CYCLES-1)) { + when (count === UInt(refill_cycles-1)) { data_sent := Bool(true) when (cmd_sent) { valid := Bool(false) } } @@ -232,12 +232,12 @@ class LLCWriteback(requestors: Int) extends Module io.mem.req_data.bits := io.data(who).bits } -class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[UInt]) extends Module +class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[UInt])(implicit conf: MemoryIFConfiguration) extends Module { val io = new Bundle { val req = Decoupled(new LLCDataReq(ways)).flip val req_data = Decoupled(new MemData).flip - val writeback = Decoupled(UInt(width = MEM_ADDR_BITS)) + val writeback = Decoupled(UInt(width = conf.addrBits)) val writeback_data = Decoupled(new MemData) val resp = Decoupled(new MemResp) val mem_resp = Valid(new MemResp).flip @@ -245,7 +245,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[UInt]) extends Modul val mem_resp_way = UInt(INPUT, log2Up(ways)) } - val data = Module(new BigMem(sets*ways*REFILL_CYCLES, 1, latency-1, leaf)(Bits(width = MEM_DATA_BITS))) + val data = Module(new BigMem(sets*ways*refill_cycles, 1, latency-1, leaf)(Bits(width = conf.dataBits))) class QEntry extends MemResp { val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] @@ -254,12 +254,12 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[UInt]) extends Modul val qReady = q.io.count <= UInt(q.entries-latency-1) val valid = Reg(init=Bool(false)) val req = Reg(io.req.bits.clone) - val count = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) - val refillCount = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) + val count = Reg(init=UInt(0, log2Up(refill_cycles))) + val refillCount = Reg(init=UInt(0, log2Up(refill_cycles))) when (data.io.in.valid && !io.mem_resp.valid) { count := count + UInt(1) - when (valid && count === UInt(REFILL_CYCLES-1)) { valid := Bool(false) } + when (valid && count === UInt(refill_cycles-1)) { valid := Bool(false) } } when (io.req.valid && io.req.ready) { valid := Bool(true); req := io.req.bits } when (io.mem_resp.valid) { refillCount := refillCount + UInt(1) } @@ -300,11 +300,11 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[UInt]) extends Modul io.writeback_data.bits := q.io.deq.bits } -class MemReqArb(n: Int) extends Module // UNTESTED +class MemReqArb(n: Int, refill_cycles: Int)(implicit conf: MemoryIFConfiguration) extends Module { val io = new Bundle { - val cpu = Vec.fill(n){new ioMem().flip} - val mem = new ioMem + val cpu = Vec.fill(n){new MemIO().flip} + val mem = new MemIO } val lock = Reg(init=Bool(false)) @@ -339,26 +339,26 @@ class MemReqArb(n: Int) extends Module // UNTESTED } io.mem.resp.ready := io.cpu(respWho).resp.ready - val unlock = Counter(io.mem.req_data.fire(), REFILL_CYCLES)._2 + val unlock = Counter(io.mem.req_data.fire(), refill_cycles)._2 when (unlock) { lock := Bool(false) } } -class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[UInt], dataLeaf: Mem[UInt]) extends Module +class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, tagLeaf: Mem[UInt], dataLeaf: Mem[UInt])(implicit conf: MemoryIFConfiguration) extends Module { val io = new Bundle { - val cpu = new ioMem().flip - val mem = new ioMemPipe + val cpu = new MemIO().flip + val mem = new MemPipeIO } - val tagWidth = MEM_ADDR_BITS - log2Up(sets) + val tagWidth = conf.addrBits - log2Up(sets) val metaWidth = tagWidth + 2 // valid + dirty val memCmdArb = Module(new Arbiter(new MemReqCmd, 2)) val dataArb = Module(new Arbiter(new LLCDataReq(ways), 2)) - val mshr = Module(new LLCMSHRFile(sets, ways, outstanding)) + val mshr = Module(new LLCMSHRFile(sets, ways, outstanding, refill_cycles)) val tags = Module(new BigMem(sets, 0, 1, tagLeaf)(Bits(width = metaWidth*ways))) - val data = Module(new LLCData(4, sets, ways, dataLeaf)) - val writeback = Module(new LLCWriteback(2)) + val data = Module(new LLCData(4, sets, ways, refill_cycles, dataLeaf)) + val writeback = Module(new LLCWriteback(2, refill_cycles)) val initCount = Reg(init=UInt(0, log2Up(sets+1))) val initialize = !initCount(log2Up(sets)) @@ -497,11 +497,11 @@ object HellaQueue } } -class DRAMSideLLCNull(numRequests: Int, refillCycles: Int) extends Module +class DRAMSideLLCNull(numRequests: Int, refillCycles: Int)(implicit conf: MemoryIFConfiguration) extends Module { val io = new Bundle { - val cpu = new ioMem().flip - val mem = new ioMemPipe + val cpu = new MemIO().flip + val mem = new MemPipeIO } val numEntries = numRequests * refillCycles diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 42611bc6..21a987fd 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -2,49 +2,56 @@ package uncore import Chisel._ import scala.math._ -trait HasMemData extends Bundle { - val data = Bits(width = MEM_DATA_BITS) +case class MemoryIFConfiguration(addrBits: Int, dataBits: Int, tagBits: Int, dataBeats: Int) + +abstract trait MemoryIFSubBundle extends Bundle { + implicit val conf: MemoryIFConfiguration + override def clone = this.getClass.getConstructors.head.newInstance(conf).asInstanceOf[this.type] } -trait HasMemAddr extends Bundle { - val addr = UInt(width = MEM_ADDR_BITS) +trait HasMemData extends MemoryIFSubBundle { + val data = Bits(width = conf.dataBits) } -trait HasMemTag extends Bundle { - val tag = UInt(width = MEM_TAG_BITS) +trait HasMemAddr extends MemoryIFSubBundle { + val addr = UInt(width = conf.addrBits) } -class MemReqCmd extends HasMemAddr with HasMemTag { +trait HasMemTag extends MemoryIFSubBundle { + val tag = UInt(width = conf.tagBits) +} + +class MemReqCmd(implicit val conf: MemoryIFConfiguration) extends HasMemAddr with HasMemTag { val rw = Bool() } -class MemResp extends HasMemData with HasMemTag +class MemResp(implicit val conf: MemoryIFConfiguration) extends HasMemData with HasMemTag -class MemData extends HasMemData +class MemData(implicit val conf: MemoryIFConfiguration) extends HasMemData -class ioMem extends Bundle { +class MemIO(implicit val conf: MemoryIFConfiguration) extends Bundle { val req_cmd = Decoupled(new MemReqCmd) val req_data = Decoupled(new MemData) val resp = Decoupled(new MemResp).flip } -class ioMemPipe extends Bundle { +class MemPipeIO(implicit val conf: MemoryIFConfiguration) extends Bundle { val req_cmd = Decoupled(new MemReqCmd) val req_data = Decoupled(new MemData) val resp = Valid(new MemResp).flip } -class ioMemSerialized(w: Int) extends Bundle +class MemSerializedIO(w: Int)(implicit val conf: MemoryIFConfiguration) extends Bundle { val req = Decoupled(Bits(width = w)) val resp = Valid(Bits(width = w)).flip } -class MemSerdes(w: Int) extends Module +class MemSerdes(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module { val io = new Bundle { - val wide = new ioMem().flip - val narrow = new ioMemSerialized(w) + val wide = new MemIO().flip + val narrow = new MemSerializedIO(w) } val abits = io.wide.req_cmd.bits.toBits.getWidth val dbits = io.wide.req_data.bits.toBits.getWidth @@ -56,7 +63,7 @@ class MemSerdes(w: Int) extends Module val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(UInt(), 5) val state = Reg(init=s_idle) val send_cnt = Reg(init=UInt(0, log2Up((max(abits, dbits)+w-1)/w))) - val data_send_cnt = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) + val data_send_cnt = Reg(init=UInt(0, log2Up(conf.dataBeats))) val adone = io.narrow.req.ready && send_cnt === UInt((abits-1)/w) val ddone = io.narrow.req.ready && send_cnt === UInt((dbits-1)/w) @@ -92,12 +99,12 @@ class MemSerdes(w: Int) extends Module } when (state === s_write_data && ddone) { data_send_cnt := data_send_cnt + UInt(1) - state := Mux(data_send_cnt === UInt(REFILL_CYCLES-1), s_idle, s_write_idle) + state := Mux(data_send_cnt === UInt(conf.dataBeats-1), s_idle, s_write_idle) send_cnt := UInt(0) } val recv_cnt = Reg(init=UInt(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) + val data_recv_cnt = Reg(init=UInt(0, log2Up(conf.dataBeats))) val resp_val = Reg(init=Bool(false)) resp_val := Bool(false) @@ -115,12 +122,12 @@ class MemSerdes(w: Int) extends Module io.wide.resp.bits := io.wide.resp.bits.fromBits(in_buf) } -class MemDesserIO(w: Int) extends Bundle { - val narrow = new ioMemSerialized(w).flip - val wide = new ioMem +class MemDesserIO(w: Int)(implicit val conf: MemoryIFConfiguration) extends Bundle { + val narrow = new MemSerializedIO(w).flip + val wide = new MemIO } -class MemDesser(w: Int) extends Module // test rig side +class MemDesser(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module // test rig side { val io = new MemDesserIO(w) val abits = io.wide.req_cmd.bits.toBits.getWidth @@ -129,7 +136,7 @@ class MemDesser(w: Int) extends Module // test rig side require(dbits >= abits && rbits >= dbits) val recv_cnt = Reg(init=UInt(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) + val data_recv_cnt = Reg(init=UInt(0, log2Up(conf.dataBeats))) val adone = io.narrow.req.valid && recv_cnt === UInt((abits-1)/w) val ddone = io.narrow.req.valid && recv_cnt === UInt((dbits-1)/w) val rdone = io.narrow.resp.valid && recv_cnt === UInt((rbits-1)/w) @@ -157,13 +164,13 @@ class MemDesser(w: Int) extends Module // test rig side } when (state === s_data && io.wide.req_data.ready) { state := s_data_recv - when (data_recv_cnt === UInt(REFILL_CYCLES-1)) { + when (data_recv_cnt === UInt(conf.dataBeats-1)) { state := s_cmd_recv } data_recv_cnt := data_recv_cnt + UInt(1) } when (rdone) { // state === s_reply - when (data_recv_cnt === UInt(REFILL_CYCLES-1)) { + when (data_recv_cnt === UInt(conf.dataBeats-1)) { state := s_cmd_recv } recv_cnt := UInt(0) @@ -177,10 +184,92 @@ class MemDesser(w: Int) extends Module // test rig side io.wide.req_data.valid := state === s_data io.wide.req_data.bits.data := in_buf >> UInt(((rbits+w-1)/w - (dbits+w-1)/w)*w) - val dataq = Module(new Queue(new MemResp, REFILL_CYCLES)) + val dataq = Module(new Queue(new MemResp, conf.dataBeats)) dataq.io.enq <> io.wide.resp dataq.io.deq.ready := recv_cnt === UInt((rbits-1)/w) io.narrow.resp.valid := dataq.io.deq.valid io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UInt(w)) } + +//Adapter betweewn an UncachedTileLinkIO and a mem controller MemIO +class MemIOUncachedTileLinkIOConverter(qDepth: Int)(implicit tlconf: TileLinkConfiguration, mifconf: MemoryIFConfiguration) extends Module { + val io = new Bundle { + val uncached = new UncachedTileLinkIO().flip + val mem = new MemIO + } + + require(tlconf.dataBits == mifconf.dataBits*mifconf.dataBeats) + //require(tlconf.clientXactIdBits <= mifconf.tagBits) + + val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth)) + val mem_data_q = Module(new Queue(new MemData, qDepth)) + val cnt_max = mifconf.dataBeats + + val cnt_out = Reg(UInt(width = log2Up(cnt_max+1))) + val active_out = Reg(init=Bool(false)) + val cmd_sent_out = Reg(init=Bool(false)) + val buf_out = Reg(Bits()) + val tag_out = Reg(Bits()) + val addr_out = Reg(Bits()) + val has_data = Reg(init=Bool(false)) + + val cnt_in = Reg(UInt(width = log2Up(cnt_max+1))) + val active_in = Reg(init=Bool(false)) + val buf_in = Reg(Bits()) + val tag_in = Reg(UInt(width = mifconf.tagBits)) + + // Decompose outgoing TL Acquires into MemIO cmd and data + when(!active_out && io.uncached.acquire.valid) { + active_out := Bool(true) + cmd_sent_out := Bool(false) + cnt_out := UInt(0) + buf_out := io.uncached.acquire.bits.payload.data + tag_out := io.uncached.acquire.bits.payload.client_xact_id + addr_out := io.uncached.acquire.bits.payload.addr + has_data := tlconf.co.needsOuterWrite(io.uncached.acquire.bits.payload.a_type, UInt(0)) + } + when(active_out) { + when(mem_cmd_q.io.enq.fire()) { + cmd_sent_out := Bool(true) + } + when(mem_data_q.io.enq.fire()) { + cnt_out := cnt_out + UInt(1) + buf_out := buf_out >> UInt(mifconf.dataBits) + } + when(cmd_sent_out && (!has_data || cnt_out === UInt(cnt_max))) { + active_out := Bool(false) + } + } + + io.uncached.acquire.ready := !active_out + mem_cmd_q.io.enq.valid := active_out && !cmd_sent_out + mem_cmd_q.io.enq.bits.rw := has_data + mem_cmd_q.io.enq.bits.tag := tag_out + mem_cmd_q.io.enq.bits.addr := addr_out + mem_data_q.io.enq.valid := active_out && has_data && cnt_out < UInt(cnt_max) + mem_data_q.io.enq.bits.data := buf_out + io.mem.req_cmd <> mem_cmd_q.io.deq + io.mem.req_data <> mem_data_q.io.deq + + // Aggregate incoming MemIO responses into TL Grants + io.mem.resp.ready := !active_in || cnt_in < UInt(cnt_max) + io.uncached.grant.valid := active_in && (cnt_in === UInt(cnt_max)) + io.uncached.grant.bits.payload := Grant(UInt(0), tag_in, UInt(0), buf_in) + when(!active_in && io.mem.resp.valid) { + active_in := Bool(true) + cnt_in := UInt(1) + buf_in := io.mem.resp.bits.data << UInt(mifconf.dataBits*(cnt_max-1)) + tag_in := io.mem.resp.bits.tag + } + when(active_in) { + when(io.uncached.grant.fire()) { + active_in := Bool(false) + } + when(io.mem.resp.fire()) { + buf_in := Cat(io.mem.resp.bits.data, buf_in(cnt_max*mifconf.dataBits-1,mifconf.dataBits)) + cnt_in := cnt_in + UInt(1) + } + } +} + diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index 613a3b65..6889bd7f 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -1,102 +1,5 @@ package uncore import Chisel._ -import scala.collection.mutable.Stack - -class PairedDataIO[M <: Data, D <: Data](mType: M, dType: D) extends Bundle { - val meta = Decoupled(mType) - val data = Decoupled(dType) - override def clone = { new PairedDataIO(mType, dType).asInstanceOf[this.type] } -} - -class PairedArbiterIO[M <: Data, D <: Data](mType: M, dType: D, n: Int) extends Bundle { - val in = Vec.fill(n){new PairedDataIO(mType, dType)}.flip - val out = new PairedDataIO(mType, dType) - val meta_chosen = UInt(OUTPUT, log2Up(n)) - val data_chosen = UInt(OUTPUT, log2Up(n)) - override def clone = { new PairedArbiterIO(mType, dType, n).asInstanceOf[this.type] } -} - -class PairedLockingRRArbiter[M <: Data, D <: Data](mType: M, dType: D, n: Int, count: Int, needsLock: Option[M => Bool] = None) extends Module { - require(isPow2(count)) - val io = new PairedArbiterIO(mType, dType, n) - val locked = if(count > 1) Reg(init=Bool(false)) else Bool(false) - val lockIdx = if(count > 1) Reg(init=UInt(n-1)) else UInt(n-1) - val grant = List.fill(n)(Bool()) - val meta_chosen = Bits(width = log2Up(n)) - - val chosen_meta_has_data = needsLock.map(_(io.in(meta_chosen).meta.bits)).getOrElse(Bool(true)) - val valid_meta_has_data = io.in(meta_chosen).meta.valid && chosen_meta_has_data - val grant_chosen_meta = !(locked && chosen_meta_has_data) - (0 until n).map(i => io.in(i).meta.ready := grant(i) && grant_chosen_meta && io.out.meta.ready) - (0 until n).map(i => io.in(i).data.ready := Mux(locked, lockIdx === UInt(i), grant(i) && valid_meta_has_data) && io.out.data.ready) - io.out.meta.valid := io.in(meta_chosen).meta.valid && grant_chosen_meta - io.out.data.valid := Mux(locked, io.in(lockIdx).data.valid, io.in(meta_chosen).data.valid && valid_meta_has_data) - io.out.meta.bits := io.in(meta_chosen).meta.bits - io.out.data.bits := Mux(locked, io.in(lockIdx).data.bits, io.in(meta_chosen).data.bits) - io.meta_chosen := meta_chosen - io.data_chosen := Mux(locked, lockIdx, meta_chosen) - - if(count > 1){ - val cnt = Reg(init=UInt(0, width = log2Up(count))) - val cnt_next = cnt + UInt(1) - when(io.out.data.fire()){ - cnt := cnt_next - when(cnt_next === UInt(0)) { - locked := Bool(false) - } - } - when(io.out.meta.fire()) { - when(needsLock.map(_(io.out.meta.bits)).getOrElse(Bool(true))) { - when(!locked) { - locked := Bool(true) - lockIdx := Vec(io.in.map{in => in.meta.fire()}).indexWhere{i: Bool => i} - } - } - } - } - val last_grant = Reg(init=Bits(0, log2Up(n))) - val ctrl = ArbiterCtrl((0 until n).map(i => io.in(i).meta.valid && UInt(i) > last_grant) ++ io.in.map(_.meta.valid)) - (0 until n).map(i => grant(i) := ctrl(i) && UInt(i) > last_grant || ctrl(i + n)) - - var choose = Bits(n-1) - for (i <- n-2 to 0 by -1) - choose = Mux(io.in(i).meta.valid, Bits(i), choose) - for (i <- n-1 to 1 by -1) - choose = Mux(io.in(i).meta.valid && UInt(i) > last_grant, Bits(i), choose) - meta_chosen := choose - - when (io.out.meta.fire()) { last_grant := meta_chosen } -} - -class PairedCrossbarIO[M <: Data, D <: Data](mType: M, dType: D)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { - val in = Vec.fill(conf.nEndpoints){new PairedDataIO(new PhysicalNetworkIO(mType),new PhysicalNetworkIO(dType))}.flip - val out = Vec.fill(conf.nEndpoints){new PairedDataIO(new PhysicalNetworkIO(mType),new PhysicalNetworkIO(dType))} -} - -class PairedCrossbar[M <: Data, D <: Data](mType: M, dType: D, count: Int, needsLock: Option[PhysicalNetworkIO[M] => Bool] = None)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { - val io = new PairedCrossbarIO(mType, dType) - - val metaRdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints){Bool()}) - val dataRdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints){Bool()}) - val rdyVecs = metaRdyVecs zip dataRdyVecs - - io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { - val rrarb = Module(new PairedLockingRRArbiter(io.in(0).meta.bits.clone, io.in(0).data.bits.clone, conf.nEndpoints, count, needsLock)) - rrarb.io.in zip io.in zip rdys._1 zip rdys._2 map { case (((arb, in), meta_rdy), data_rdy) => { - arb.meta.valid := in.meta.valid && (in.meta.bits.header.dst === UInt(i)) - arb.meta.bits := in.meta.bits - meta_rdy := arb.meta.ready && (in.meta.bits.header.dst === UInt(i)) - arb.data.valid := in.data.valid && (in.data.bits.header.dst === UInt(i)) - arb.data.bits := in.data.bits - data_rdy := arb.data.ready && (in.data.bits.header.dst === UInt(i)) - }} - out <> rrarb.io.out - }} - for(i <- 0 until conf.nEndpoints) { - io.in(i).meta.ready := rdyVecs.map(r => r._1(i)).reduceLeft(_||_) - io.in(i).data.ready := rdyVecs.map(r => r._2(i)).reduceLeft(_||_) - } -} case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) @@ -147,7 +50,13 @@ class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { val dst = UInt(width = conf.idBits) } -object FIFOedLogicalNetworkIOWrapper { +class LogicalNetworkIO[T <: Data](dType: T)(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val header = new LogicalHeader + val payload = dType.clone + override def clone = { new LogicalNetworkIO(dType).asInstanceOf[this.type] } +} + +object DecoupledLogicalNetworkIOWrapper { def apply[T <: Data](in: DecoupledIO[T], src: UInt = UInt(0), dst: UInt = UInt(0))(implicit conf: LogicalNetworkConfiguration) = { val out = Decoupled(new LogicalNetworkIO(in.bits.clone)).asDirectionless out.valid := in.valid @@ -159,7 +68,7 @@ object FIFOedLogicalNetworkIOWrapper { } } -object FIFOedLogicalNetworkIOUnwrapper { +object DecoupledLogicalNetworkIOUnwrapper { def apply[T <: Data](in: DecoupledIO[LogicalNetworkIO[T]])(implicit conf: LogicalNetworkConfiguration) = { val out = Decoupled(in.bits.payload.clone).asDirectionless out.valid := in.valid @@ -168,9 +77,3 @@ object FIFOedLogicalNetworkIOUnwrapper { out } } - -class LogicalNetworkIO[T <: Data](dType: T)(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val header = new LogicalHeader - val payload = dType.clone - override def clone = { new LogicalNetworkIO(dType).asInstanceOf[this.type] } -} diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index cd50213f..6f877856 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -35,18 +35,24 @@ object Acquire acq.a_type := a_type acq.addr := addr acq.client_xact_id := client_xact_id + acq.data := Bits(0) acq.write_mask := Bits(0) acq.subword_addr := Bits(0) acq.atomic_opcode := Bits(0) acq } - def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, write_mask: Bits)(implicit conf: TileLinkConfiguration): Acquire = { + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt)(implicit conf: TileLinkConfiguration): Acquire = { val acq = apply(a_type, addr, client_xact_id) + acq.data := data + acq + } + def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, write_mask: Bits, data: UInt)(implicit conf: TileLinkConfiguration): Acquire = { + val acq = apply(a_type, addr, client_xact_id, data) acq.write_mask := write_mask acq } - def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt)(implicit conf: TileLinkConfiguration): Acquire = { - val acq = apply(a_type, addr, client_xact_id) + def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt, data: UInt)(implicit conf: TileLinkConfiguration): Acquire = { + val acq = apply(a_type, addr, client_xact_id, data) acq.subword_addr := subword_addr acq.atomic_opcode := atomic_opcode acq @@ -57,15 +63,17 @@ object Acquire acq } } -class Acquire(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId { + +class Acquire(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage + with HasPhysicalAddress + with HasClientTransactionId + with HasTileLinkData { val a_type = UInt(width = conf.co.acquireTypeWidth) val write_mask = Bits(width = ACQUIRE_WRITE_MASK_BITS) val subword_addr = Bits(width = ACQUIRE_SUBWORD_ADDR_BITS) val atomic_opcode = Bits(width = ACQUIRE_ATOMIC_OP_BITS) } -class AcquireData(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasTileLinkData - object Probe { def apply(p_type: UInt, addr: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { @@ -76,16 +84,20 @@ object Probe prb } } -class Probe(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage with HasPhysicalAddress with HasMasterTransactionId { + +class Probe(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage + with HasPhysicalAddress + with HasMasterTransactionId { val p_type = UInt(width = conf.co.probeTypeWidth) } object Release { - def apply(r_type: UInt, addr: UInt)(implicit conf: TileLinkConfiguration) = { + def apply(r_type: UInt, addr: UInt, data: UInt)(implicit conf: TileLinkConfiguration) = { val rel = new Release rel.r_type := r_type rel.addr := addr + rel.data := data rel } def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { @@ -94,16 +106,45 @@ object Release rel.addr := addr rel.client_xact_id := client_xact_id rel.master_xact_id := master_xact_id + rel.data := UInt(0) + rel + } + def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt)(implicit conf: TileLinkConfiguration): Release = { + val rel = apply(r_type, addr, client_xact_id, master_xact_id) + rel.data := data rel } } -class Release(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasMasterTransactionId { + +class Release(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage + with HasPhysicalAddress + with HasClientTransactionId + with HasMasterTransactionId + with HasTileLinkData { val r_type = UInt(width = conf.co.releaseTypeWidth) } -class ReleaseData(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasTileLinkData +object Grant +{ + def apply(g_type: UInt, client_xact_id: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { + val gnt = new Grant + gnt.g_type := g_type + gnt.client_xact_id := client_xact_id + gnt.master_xact_id := master_xact_id + gnt.data := UInt(0) + gnt + } + def apply(g_type: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt)(implicit conf: TileLinkConfiguration): Grant = { + val gnt = apply(g_type, client_xact_id, master_xact_id) + gnt.data := data + gnt + } +} -class Grant(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage with HasTileLinkData with HasClientTransactionId with HasMasterTransactionId { +class Grant(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage + with HasTileLinkData + with HasClientTransactionId + with HasMasterTransactionId { val g_type = UInt(width = conf.co.grantTypeWidth) } @@ -112,7 +153,7 @@ class GrantAck(implicit val conf: TileLinkConfiguration) extends ClientSourcedMe class UncachedTileLinkIO(implicit conf: TileLinkConfiguration) extends Bundle { implicit val ln = conf.ln - val acquire = new PairedDataIO(new LogicalNetworkIO(new Acquire), new LogicalNetworkIO(new AcquireData)) + val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip val grant_ack = new DecoupledIO(new LogicalNetworkIO(new GrantAck)) override def clone = { new UncachedTileLinkIO().asInstanceOf[this.type] } @@ -120,7 +161,7 @@ class UncachedTileLinkIO(implicit conf: TileLinkConfiguration) extends Bundle { class TileLinkIO(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIO()(conf) { val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip - val release = new PairedDataIO(new LogicalNetworkIO(new Release), new LogicalNetworkIO(new ReleaseData)) + val release = new DecoupledIO(new LogicalNetworkIO(new Release)) override def clone = { new TileLinkIO().asInstanceOf[this.type] } } @@ -129,32 +170,28 @@ abstract class TileLinkArbiterLike(val arbN: Int)(implicit conf: TileLinkConfigu type MasterSourcedWithId = MasterSourcedMessage with HasClientTransactionId type ClientSourcedWithId = ClientSourcedMessage with HasClientTransactionId - type ClientSourcedWithData = ClientSourcedMessage with HasTileLinkData def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int): Bits def masterSourcedClientXactId(in: MasterSourcedWithId): Bits def arbIdx(in: MasterSourcedWithId): UInt - def hookupClientSource[M <: ClientSourcedWithId, D <: ClientSourcedWithData] - (ins: Seq[PairedDataIO[LogicalNetworkIO[M],LogicalNetworkIO[D]]], - out: PairedDataIO[LogicalNetworkIO[M],LogicalNetworkIO[D]]) { + def hookupClientSource[M <: ClientSourcedWithId] + (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], + out: DecoupledIO[LogicalNetworkIO[M]]) { def hasData(m: LogicalNetworkIO[M]) = co.messageHasData(m.payload) - val arb = Module(new PairedLockingRRArbiter(out.meta.bits.clone, out.data.bits.clone, - arbN, REFILL_CYCLES, hasData _)) + val arb = Module(new RRArbiter(out.bits.clone, arbN)) out <> arb.io.out ins.zipWithIndex.zip(arb.io.in).map{ case ((req,id), arb) => { - arb.data <> req.data - arb.meta.valid := req.meta.valid - arb.meta.bits := req.meta.bits - arb.meta.bits.payload := req.meta.bits.payload - arb.meta.bits.payload.client_xact_id := clientSourcedClientXactId(req.meta.bits.payload, id) - req.meta.ready := arb.meta.ready + arb.valid := req.valid + arb.bits := req.bits + arb.bits.payload.client_xact_id := clientSourcedClientXactId(req.bits.payload, id) + req.ready := arb.ready }} } def hookupMasterSource[M <: MasterSourcedWithId] (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], - out: DecoupledIO[LogicalNetworkIO[M]]) { + out: DecoupledIO[LogicalNetworkIO[M]]) { out.ready := Bool(false) for (i <- 0 until arbN) { ins(i).valid := Bool(false) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 67dfde85..3e162583 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -31,16 +31,12 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.client - alloc_arb.io.in(i).valid := t.acquire.meta.ready - t.acquire.meta.bits := acquire.meta.bits - t.acquire.meta.valid := alloc_arb.io.in(i).ready - - t.acquire.data.bits := acquire.data.bits - t.acquire.data.valid := acquire.data.valid + alloc_arb.io.in(i).valid := t.acquire.ready + t.acquire.bits := acquire.bits + t.acquire.valid := alloc_arb.io.in(i).ready } - acquire.meta.ready := trackerList.map(_.io.client.acquire.meta.ready).reduce(_||_) && !block_acquires - acquire.data.ready := trackerList.map(_.io.client.acquire.data.ready).reduce(_||_) - alloc_arb.io.out.ready := acquire.meta.valid && !block_acquires + acquire.ready := trackerList.map(_.io.client.acquire.ready).reduce(_||_) && !block_acquires + alloc_arb.io.out.ready := acquire.valid && !block_acquires // Handle probe request generation val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) @@ -49,21 +45,18 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration // Handle releases, which might be voluntary and might have data val release = io.client.release - val voluntary = co.isVoluntary(release.meta.bits.payload) + val voluntary = co.isVoluntary(release.bits.payload) val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) val block_releases = Bool(false) val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b} //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response - val release_idx = Mux(voluntary, UInt(0), release.meta.bits.payload.master_xact_id) + val release_idx = Mux(voluntary, UInt(0), release.bits.payload.master_xact_id) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.client - t.release.meta.bits := release.meta.bits - t.release.meta.valid := release.meta.valid && (release_idx === UInt(i)) && !block_releases - t.release.data.bits := release.data.bits - t.release.data.valid := release.data.valid + t.release.bits := release.bits + t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases } - release.meta.ready := Vec(trackerList.map(_.io.client.release.meta.ready)).read(release_idx) && !block_releases - release.data.ready := trackerList.map(_.io.client.release.data.ready).reduce(_||_) + release.ready := Vec(trackerList.map(_.io.client.release.ready)).read(release_idx) && !block_releases // Reply to initial requestor val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) @@ -83,7 +76,7 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration } -abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) extends Module with OuterRequestGenerator { +abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) extends Module { implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val io = new Bundle { val client = (new TileLinkIO).flip @@ -92,63 +85,56 @@ abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) exten val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) } + + val c_acq = io.client.acquire.bits + val c_rel = io.client.release.bits + val c_gnt = io.client.grant.bits + val c_ack = io.client.grant_ack.bits + val m_gnt = io.master.grant.bits + } class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) val xact = Reg{ new Release } - val init_client_id_ = Reg(init=UInt(0, width = log2Up(ln.nClients))) - val release_data_needs_write = Reg(init=Bool(false)) - val mem_cmd_sent = Reg(init=Bool(false)) - val cmd_to_write = Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId)) + val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val incoming_rel = io.client.release.bits io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.meta.bits.payload.addr) && (state != s_idle) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, incoming_rel.payload.addr) && + (state != s_idle) io.master.grant.ready := Bool(false) - io.master.acquire.meta.valid := Bool(false) - io.master.acquire.meta.bits.payload := cmd_to_write - //TODO io.master.acquire.bits.header.dst - io.master.acquire.meta.bits.header.src := UInt(bankId) - io.master.acquire.data.valid := Bool(false) - io.master.acquire.data.bits.payload.data := UInt(0) - //TODO io.master.acquire_data.bits.header.dst - io.master.acquire.data.bits.header.src := UInt(bankId) - io.client.acquire.meta.ready := Bool(false) - io.client.acquire.data.ready := Bool(false) + io.master.acquire.valid := Bool(false) + io.master.acquire.bits.header.src := UInt(bankId) + //io.master.acquire.bits.header.dst TODO + io.master.acquire.bits.payload := Acquire(co.getUncachedWriteAcquireType, + xact.addr, + UInt(trackerId), + xact.data) + io.client.acquire.ready := Bool(false) io.client.probe.valid := Bool(false) - io.client.release.meta.ready := Bool(false) - io.client.release.data.ready := Bool(false) // DNC + io.client.release.ready := Bool(false) io.client.grant.valid := Bool(false) - io.client.grant.bits.payload.g_type := co.getGrantType(xact, UInt(0)) - io.client.grant.bits.payload.client_xact_id := xact.client_xact_id - io.client.grant.bits.payload.master_xact_id := UInt(trackerId) - io.client.grant.bits.payload.data := UInt(0) - io.client.grant.bits.header.dst := init_client_id_ io.client.grant.bits.header.src := UInt(bankId) + io.client.grant.bits.header.dst := init_client_id + io.client.grant.bits.payload := Grant(co.getGrantType(xact, UInt(0)), + xact.client_xact_id, + UInt(trackerId)) switch (state) { is(s_idle) { - io.client.release.meta.ready := Bool(true) - when( io.client.release.meta.valid ) { - xact := io.client.release.meta.bits.payload - init_client_id_ := io.client.release.meta.bits.header.src - release_data_needs_write := co.messageHasData(io.client.release.meta.bits.payload) - mem_cnt := UInt(0) - mem_cmd_sent := Bool(false) - state := s_mem + io.client.release.ready := Bool(true) + when( io.client.release.valid ) { + xact := incoming_rel.payload + init_client_id := incoming_rel.header.src + state := Mux(co.messageHasData(incoming_rel.payload), s_mem, s_ack) } } is(s_mem) { - when (release_data_needs_write) { - doOuterReqWrite(io.master.acquire, - io.client.release.data, - cmd_to_write, - release_data_needs_write, - mem_cmd_sent, - init_client_id_) - } . otherwise { state := s_ack } + io.master.acquire.valid := Bool(true) + when(io.master.acquire.ready) { state := s_ack } } is(s_ack) { io.client.grant.valid := Bool(true) @@ -158,172 +144,138 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Cohe } class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { - val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(UInt(), 5) + val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) val state = Reg(init=s_idle) val xact = Reg{ new Acquire } - val init_client_id_ = Reg(init=UInt(0, width = log2Up(ln.nClients))) - val release_data_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) //TODO: Will need id reg for merged release xacts - val init_sharer_cnt_ = Reg(init=UInt(0, width = log2Up(ln.nClients))) - val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt_) + + val init_sharer_cnt = Reg(init=UInt(0, width = log2Up(ln.nClients))) val release_count = if (ln.nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(ln.nClients))) val probe_flags = Reg(init=Bits(0, width = ln.nClients)) val curr_p_id = PriorityEncoder(probe_flags) - val x_needs_read = Reg(init=Bool(false)) - val acquire_data_needs_write = Reg(init=Bool(false)) - val release_data_needs_write = Reg(init=Bool(false)) - val cmd_to_write = Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId)) - val cmd_to_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) - val a_w_mem_cmd_sent = Reg(init=Bool(false)) - val r_w_mem_cmd_sent = Reg(init=Bool(false)) + + val pending_outer_write = co.messageHasData(xact) + val pending_outer_read = co.needsOuterRead(xact.a_type, UInt(0)) + val outer_write_acq = Acquire(co.getUncachedWriteAcquireType, + xact.addr, UInt(trackerId), xact.data) + val outer_write_rel = Acquire(co.getUncachedWriteAcquireType, + xact.addr, UInt(trackerId), c_rel.payload.data) + val outer_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) + val probe_initial_flags = Bits(width = ln.nClients) probe_initial_flags := Bits(0) if (ln.nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence val probe_self = Bool(true) //co.needsSelfProbe(io.client.acquire.bits.payload) - val myflag = Mux(probe_self, Bits(0), UIntToOH(io.client.acquire.meta.bits.header.src(log2Up(ln.nClients)-1,0))) + val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(ln.nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } - io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, io.client.acquire.meta.bits.payload.addr) && (state != s_idle) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.meta.bits.payload.addr) && (state != s_idle) - io.master.acquire.meta.valid := Bool(false) - io.master.acquire.meta.bits.payload := Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) - io.master.acquire.meta.bits.header.src := UInt(bankId) - io.master.acquire.data.valid := Bool(false) - io.master.acquire.data.bits.payload.data := UInt(0) - io.master.acquire.data.bits.header.src := UInt(bankId) - io.client.probe.valid := Bool(false) - io.client.probe.bits.payload.p_type := co.getProbeType(xact.a_type, UInt(0)) - io.client.probe.bits.payload.master_xact_id := UInt(trackerId) - io.client.probe.bits.payload.addr := xact.addr - io.client.probe.bits.header.dst := UInt(0) - io.client.probe.bits.header.src := UInt(bankId) - io.client.grant.bits.payload.data := io.master.grant.bits.payload.data - io.client.grant.bits.payload.g_type := grant_type - io.client.grant.bits.payload.client_xact_id := xact.client_xact_id - io.client.grant.bits.payload.master_xact_id := UInt(trackerId) - io.client.grant.bits.header.dst := init_client_id_ - io.client.grant.bits.header.src := UInt(bankId) - io.client.grant.valid := (io.master.grant.valid && (UInt(trackerId) === io.master.grant.bits.payload.client_xact_id)) - io.client.acquire.meta.ready := Bool(false) - io.client.acquire.data.ready := Bool(false) - io.client.release.meta.ready := Bool(false) - io.client.release.data.ready := Bool(false) + io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) + + io.master.acquire.valid := Bool(false) + io.master.acquire.bits.header.src := UInt(bankId) + //io.master.acquire.bits.header.dst TODO + io.master.acquire.bits.payload := outer_read io.master.grant.ready := io.client.grant.ready + io.client.probe.valid := Bool(false) + io.client.probe.bits.header.src := UInt(bankId) + io.client.probe.bits.header.dst := curr_p_id + io.client.probe.bits.payload := Probe(co.getProbeType(xact.a_type, UInt(0)), + xact.addr, + UInt(trackerId)) + + val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt) + io.client.grant.valid := Bool(false) + io.client.grant.bits.header.src := UInt(bankId) + io.client.grant.bits.header.dst := init_client_id + io.client.grant.bits.payload := Grant(grant_type, + xact.client_xact_id, + UInt(trackerId), + m_gnt.payload.data) + + io.client.acquire.ready := Bool(false) + io.client.release.ready := Bool(false) + switch (state) { is(s_idle) { - io.client.acquire.meta.ready := Bool(true) - when( io.client.acquire.meta.valid ) { - xact := io.client.acquire.meta.bits.payload - init_client_id_ := io.client.acquire.meta.bits.header.src - init_sharer_cnt_ := UInt(ln.nClients) // TODO: Broadcast only - acquire_data_needs_write := co.messageHasData(io.client.acquire.meta.bits.payload) - x_needs_read := co.needsOuterRead(io.client.acquire.meta.bits.payload.a_type, UInt(0)) + io.client.acquire.ready := Bool(true) + val needs_outer_write = co.messageHasData(c_acq.payload) + val needs_outer_read = co.needsOuterRead(c_acq.payload.a_type, UInt(0)) + when( io.client.acquire.valid ) { + xact := c_acq.payload + init_client_id := c_acq.header.src + init_sharer_cnt := UInt(ln.nClients) // TODO: Broadcast only probe_flags := probe_initial_flags - mem_cnt := UInt(0) - r_w_mem_cmd_sent := Bool(false) - a_w_mem_cmd_sent := Bool(false) if(ln.nClients > 1) { release_count := PopCount(probe_initial_flags) - state := Mux(probe_initial_flags.orR, s_probe, s_mem) - } else state := s_mem + state := Mux(probe_initial_flags.orR, s_probe, + Mux(needs_outer_write, s_mem_write, + Mux(needs_outer_read, s_mem_read, s_make_grant))) + } else state := Mux(needs_outer_write, s_mem_write, + Mux(needs_outer_read, s_mem_read, s_make_grant)) } } is(s_probe) { - when(probe_flags.orR) { - io.client.probe.valid := Bool(true) - io.client.probe.bits.header.dst := curr_p_id - } + // Generate probes + io.client.probe.valid := probe_flags.orR when(io.client.probe.ready) { probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) } - io.client.release.meta.ready := Bool(true) - when(io.client.release.meta.valid) { - if(ln.nClients > 1) release_count := release_count - UInt(1) - when(release_count === UInt(1)) { - state := s_mem - } - when( co.messageHasData(io.client.release.meta.bits.payload)) { - release_data_needs_write := Bool(true) - release_data_client_id := io.client.release.meta.bits.header.src + + // Handle releases, which may have data to be written back + when(io.client.release.valid) { + when(co.messageHasData(c_rel.payload)) { + io.master.acquire.valid := Bool(true) + io.master.acquire.bits.payload := outer_write_rel + when(io.master.acquire.ready) { + io.client.release.ready := Bool(true) + if(ln.nClients > 1) release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } + } + } .otherwise { + io.client.release.ready := Bool(true) + if(ln.nClients > 1) release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } } } - when (release_data_needs_write) { - doOuterReqWrite(io.master.acquire, - io.client.release.data, - cmd_to_write, - release_data_needs_write, - r_w_mem_cmd_sent, - release_data_client_id) + } + is(s_mem_read) { + io.master.acquire.valid := Bool(true) + io.master.acquire.bits.payload := outer_read + when(io.master.acquire.ready) { + state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) } } - is(s_mem) { - when (release_data_needs_write) { - doOuterReqWrite(io.master.acquire, - io.client.release.data, - cmd_to_write, - release_data_needs_write, - r_w_mem_cmd_sent, - release_data_client_id) - } . elsewhen(acquire_data_needs_write) { - doOuterReqWrite(io.master.acquire, - io.client.acquire.data, - cmd_to_write, - acquire_data_needs_write, - a_w_mem_cmd_sent, - init_client_id_) - } . elsewhen (x_needs_read) { - doOuterReqRead(io.master.acquire, cmd_to_read, x_needs_read) - } . otherwise { - state := Mux(co.requiresDatalessGrant(xact.a_type, UInt(0)), s_ack, - Mux(co.requiresAckForGrant(io.client.grant.bits.payload.g_type), s_busy, s_idle)) + is(s_mem_write) { + io.master.acquire.valid := Bool(true) + io.master.acquire.bits.payload := outer_write_acq + when(io.master.acquire.ready) { + state := Mux(pending_outer_read, s_mem_read, s_make_grant) } } - is(s_ack) { + is(s_make_grant) { io.client.grant.valid := Bool(true) - when(io.client.grant.ready) { state := Mux(co.requiresAckForGrant(io.client.grant.bits.payload.g_type), s_busy, s_idle) } + when(io.client.grant.ready) { + state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) + } } is(s_busy) { // Nothing left to do but wait for transaction to complete - when (io.client.grant_ack.valid && io.client.grant_ack.bits.payload.master_xact_id === UInt(trackerId)) { + when(io.master.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { + io.client.grant.valid := Bool(true) + } + when(io.client.grant_ack.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { state := s_idle } } } } - -abstract trait OuterRequestGenerator { - val mem_cnt = Reg(init=UInt(0, width = log2Up(REFILL_CYCLES))) - val mem_cnt_next = mem_cnt + UInt(1) - - def doOuterReqWrite[T <: HasTileLinkData](master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], client_data: DecoupledIO[LogicalNetworkIO[T]], cmd: Acquire, trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UInt) { - val do_write = client_data.valid && (client_data.bits.header.src === desired_client_data_src_id) - master_acq.meta.bits.payload := cmd - master_acq.data.bits.payload := client_data.bits.payload - when(master_acq.meta.fire()) { - cmd_sent := Bool(true) - } - when (do_write) { - master_acq.meta.valid := !cmd_sent - when (master_acq.meta.ready || cmd_sent) { - master_acq.data.valid := client_data.valid - when(master_acq.data.ready) { - client_data.ready:= Bool(true) - mem_cnt := mem_cnt_next - when(mem_cnt === UInt(REFILL_CYCLES-1)) { - trigger := Bool(false) - } - } - } - } - } - - def doOuterReqRead(master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], cmd: Acquire, trigger: Bool) { - master_acq.meta.valid := Bool(true) - master_acq.meta.bits.payload := cmd - when(master_acq.meta.ready) { - trigger := Bool(false) - } - } -} From b1df49ba30a9aed51527bba52afc6e151707ce2d Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 1 Apr 2014 17:14:45 -0700 Subject: [PATCH 215/374] removed AddressSpaceConstants, CacheConstants, and TileLinkSizeConstants --- uncore/src/main/scala/consts.scala | 28 --------------------------- uncore/src/main/scala/htif.scala | 10 +++++----- uncore/src/main/scala/memserdes.scala | 5 +++++ uncore/src/main/scala/package.scala | 5 +---- uncore/src/main/scala/tilelink.scala | 10 +++++----- 5 files changed, 16 insertions(+), 42 deletions(-) diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index b0cdcdd6..64d6c3a7 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -2,7 +2,6 @@ package uncore package constants import Chisel._ -import scala.math.max object MemoryOpConstants extends MemoryOpConstants trait MemoryOpConstants { @@ -43,30 +42,3 @@ trait MemoryOpConstants { def isWrite(cmd: Bits) = cmd === M_XWR || cmd === M_XSC || isAMO(cmd) def isWriteIntent(cmd: Bits) = isWrite(cmd) || cmd === M_PFW || cmd === M_XLR } - -object AddressConstants extends AddressConstants -trait AddressConstants { - val PADDR_BITS = 32 - val VADDR_BITS = 43; - val PGIDX_BITS = 13; - val PPN_BITS = PADDR_BITS-PGIDX_BITS; - val VPN_BITS = VADDR_BITS-PGIDX_BITS; - val ASID_BITS = 7; - val PERM_BITS = 6; -} - -object CacheConstants extends CacheConstants -trait CacheConstants { - val CACHE_DATA_SIZE_IN_BYTES = 1 << 6 - val OFFSET_BITS = log2Up(CACHE_DATA_SIZE_IN_BYTES) -} - -trait TileLinkSizeConstants { - val ACQUIRE_WRITE_MASK_BITS = 6 - val ACQUIRE_SUBWORD_ADDR_BITS = 3 - val ACQUIRE_ATOMIC_OP_BITS = 4 -} - -trait MemoryInterfaceConstants extends - CacheConstants with - AddressConstants diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index c4d0ffb0..9ea26f86 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -41,7 +41,7 @@ class SCRIO(n: Int) extends Bundle val wdata = Bits(OUTPUT, 64) } -class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfiguration) extends Module +class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: TileLinkConfiguration) extends Module { implicit val (ln, co) = (conf.ln, conf.co) val nTiles = ln.nClients-1 // This HTIF is itself a TileLink client @@ -75,7 +75,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati when (rx_count === UInt(short_request_bits/w-1)) { cmd := next_cmd size := rx_shifter_in(15,4) - pos := rx_shifter_in(15,4+OFFSET_BITS-3) + pos := rx_shifter_in(15,4+offsetBits-3) seqno := rx_shifter_in(23,16) addr := rx_shifter_in(63,24) } @@ -95,7 +95,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati val pcr_coreid = addr(log2Up(nTiles)-1+20+1,20) val pcr_wdata = packet_ram(0) - val bad_mem_packet = size(OFFSET_BITS-1-3,0).orR || addr(OFFSET_BITS-1-3,0).orR + val bad_mem_packet = size(offsetBits-1-3,0).orR || addr(offsetBits-1-3,0).orR val nack = Mux(cmd === cmd_readmem || cmd === cmd_writemem, bad_mem_packet, Mux(cmd === cmd_readcr || cmd === cmd_writecr, size != UInt(1), Bool(true))) @@ -157,7 +157,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati when (state === state_mem_finish && io.mem.grant_ack.ready) { state := Mux(cmd === cmd_readmem || pos === UInt(1), state_tx, state_rx) pos := pos - UInt(1) - addr := addr + UInt(1 << OFFSET_BITS-3) + addr := addr + UInt(1 << offsetBits-3) } when (state === state_tx && tx_done) { when (tx_word_count === tx_size) { @@ -176,7 +176,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati mem_req_data = Cat(packet_ram(idx), mem_req_data) } acq_q.io.enq.valid := state === state_mem_rreq || state === state_mem_wreq - val init_addr = addr.toUInt >> UInt(OFFSET_BITS-3) + val init_addr = addr.toUInt >> UInt(offsetBits-3) acq_q.io.enq.bits := Mux(cmd === cmd_writemem, Acquire(co.getUncachedWriteAcquireType, init_addr, UInt(0)), Acquire(co.getUncachedReadAcquireType, init_addr, UInt(0))) diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 21a987fd..dea3e801 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -2,6 +2,11 @@ package uncore import Chisel._ import scala.math._ +case class AddressSpaceConfiguration(paddrBits: Int, vaddrBits: Int, pgIdxBits: Int, asidBits: Int, permBits:Int) { + val ppnBits = paddrBits - pgIdxBits + val vpnBits = vaddrBits - pgIdxBits +} + case class MemoryIFConfiguration(addrBits: Int, dataBits: Int, tagBits: Int, dataBeats: Int) abstract trait MemoryIFSubBundle extends Bundle { diff --git a/uncore/src/main/scala/package.scala b/uncore/src/main/scala/package.scala index 4ac83d0b..501d511d 100644 --- a/uncore/src/main/scala/package.scala +++ b/uncore/src/main/scala/package.scala @@ -1,7 +1,4 @@ -package object uncore extends - uncore.constants.MemoryOpConstants with - uncore.constants.TileLinkSizeConstants with - uncore.constants.MemoryInterfaceConstants +package object uncore extends uncore.constants.MemoryOpConstants { implicit def toOption[A](a: A) = Option(a) } diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 6f877856..451c5844 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1,7 +1,7 @@ package uncore import Chisel._ -case class TileLinkConfiguration(co: CoherencePolicyWithUncached, ln: LogicalNetworkConfiguration, masterXactIdBits: Int, clientXactIdBits: Int, dataBits: Int) +case class TileLinkConfiguration(co: CoherencePolicyWithUncached, ln: LogicalNetworkConfiguration, addrBits: Int, masterXactIdBits: Int, clientXactIdBits: Int, dataBits: Int, writeMaskBits: Int, wordAddrBits: Int, atomicOpBits: Int) abstract trait TileLinkSubBundle extends Bundle { implicit val conf: TileLinkConfiguration @@ -9,7 +9,7 @@ abstract trait TileLinkSubBundle extends Bundle { } trait HasPhysicalAddress extends TileLinkSubBundle { - val addr = UInt(width = PADDR_BITS - OFFSET_BITS) + val addr = UInt(width = conf.addrBits) } trait HasClientTransactionId extends TileLinkSubBundle { @@ -69,9 +69,9 @@ class Acquire(implicit val conf: TileLinkConfiguration) extends ClientSourcedMes with HasClientTransactionId with HasTileLinkData { val a_type = UInt(width = conf.co.acquireTypeWidth) - val write_mask = Bits(width = ACQUIRE_WRITE_MASK_BITS) - val subword_addr = Bits(width = ACQUIRE_SUBWORD_ADDR_BITS) - val atomic_opcode = Bits(width = ACQUIRE_ATOMIC_OP_BITS) + val write_mask = Bits(width = conf.writeMaskBits) + val subword_addr = Bits(width = conf.wordAddrBits) + val atomic_opcode = Bits(width = conf.atomicOpBits) } object Probe From bf2ff7804eda45a6a690a32a11c39cafe9481539 Mon Sep 17 00:00:00 2001 From: Jim Lawson Date: Thu, 17 Apr 2014 17:01:40 -0700 Subject: [PATCH 216/374] Add chisel-dependent.sbt for -DchiselVersion="latest.release" If -DchiselVersion is specified on the command line, add the appropriate chisel library to libraryDependencies. --- uncore/chisel-dependent.sbt | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 uncore/chisel-dependent.sbt diff --git a/uncore/chisel-dependent.sbt b/uncore/chisel-dependent.sbt new file mode 100644 index 00000000..fa8d5490 --- /dev/null +++ b/uncore/chisel-dependent.sbt @@ -0,0 +1,6 @@ +// Provide a managed dependency on chisel if -DchiselVersion="" is +// supplied on the command line. + +val chiselVersion = System.getProperty("chiselVersion", "None") + +libraryDependencies ++= ( if (chiselVersion != "None" ) ("edu.berkeley.cs" %% "chisel" % chiselVersion) :: Nil; else Nil) From 5613dc7d1ba5278cc9ee00e59242f2624c9fea5d Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 16 Apr 2014 18:14:12 -0700 Subject: [PATCH 217/374] replaced Lists with Vecs --- uncore/src/main/scala/coherence.scala | 107 ++++++++++++-------------- 1 file changed, 51 insertions(+), 56 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index c9d0f4eb..327e31e5 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -52,8 +52,6 @@ abstract class CoherencePolicy { def requiresAckForGrant(g_type: UInt): Bool def requiresAckForRelease(r_type: UInt): Bool def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool - - def uIntListContains(list: List[UInt], elem: UInt): Bool = list.map(elem === _).reduceLeft(_||_) } trait UncachedTransactions { @@ -96,10 +94,9 @@ class ThreeStateIncoherence extends IncoherentPolicy { val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(UInt(), nAcquireTypes) val releaseVoluntaryInvalidateData :: releaseInvalidateAck :: Nil = Enum(UInt(), nReleaseTypes) val grantVoluntaryAck :: grantData :: grantAck :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeList = List() - val hasDataAcquireTypeList = List(acquireWriteback) - val hasDataReleaseTypeList = List(acquireWriteback) - val hasDataGrantTypeList = List(grantData) + val hasDataAcquireTypeVec = Vec(acquireWriteback) + val hasDataReleaseTypeVec = Vec(acquireWriteback) + val hasDataGrantTypeVec = Vec(grantData) def isHit ( cmd: UInt, state: UInt): Bool = (state === tileClean || state === tileDirty) def isValid (state: UInt): Bool = state != tileInvalid @@ -135,13 +132,13 @@ class ThreeStateIncoherence extends IncoherentPolicy { def getReleaseTypeOnVoluntaryWriteback(): UInt = releaseVoluntaryInvalidateData def messageHasData( msg: SourcedMessage ) = msg match { - case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) + case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) + case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) case rel: Release => Bool(false) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant) = (reply.g_type === grantData) - def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = Bool(false) } class MICoherence extends CoherencePolicyWithUncached { @@ -160,10 +157,10 @@ class MICoherence extends CoherencePolicyWithUncached { val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseCopyData) - val hasDataGrantTypeList = List(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) + val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseCopyData) + val hasDataGrantTypeVec = Vec(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) def isHit (cmd: UInt, state: UInt): Bool = state != tileInvalid def isValid (state: UInt): Bool = state != tileInvalid @@ -231,15 +228,15 @@ class MICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uIntListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) + case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) @@ -302,10 +299,10 @@ class MEICoherence extends CoherencePolicyWithUncached { val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeList = List(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) + val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val hasDataGrantTypeVec = Vec(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) def isHit (cmd: UInt, state: UInt): Bool = state != tileInvalid def isValid (state: UInt): Bool = state != tileInvalid @@ -386,15 +383,15 @@ class MEICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uIntListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) + case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) @@ -460,10 +457,10 @@ class MSICoherence extends CoherencePolicyWithUncached { val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) + val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) def isHit (cmd: UInt, state: UInt): Bool = { Mux(isWriteIntent(cmd), (state === tileExclusiveDirty), @@ -550,15 +547,15 @@ class MSICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uIntListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) + case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) @@ -621,10 +618,10 @@ class MESICoherence extends CoherencePolicyWithUncached { val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) + val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) def isHit (cmd: UInt, state: UInt): Bool = { Mux(isWriteIntent(cmd), (state === tileExclusiveClean || state === tileExclusiveDirty), @@ -711,15 +708,15 @@ class MESICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uIntListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) + case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) @@ -785,13 +782,13 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(UInt(), nReleaseTypes) val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadMigratory, grantReadWordUncached, grantAtomicUncached) - val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) + val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadMigratory, grantReadWordUncached, grantAtomicUncached) + val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) def isHit (cmd: UInt, state: UInt): Bool = { - Mux(isWriteIntent(cmd), uIntListContains(List(tileExclusiveClean, tileExclusiveDirty, tileMigratoryClean, tileMigratoryDirty), state), (state != tileInvalid)) + Mux(isWriteIntent(cmd), Vec(tileExclusiveClean, tileExclusiveDirty, tileMigratoryClean, tileMigratoryDirty).contains(state), (state != tileInvalid)) } def isValid (state: UInt): Bool = { state != tileInvalid @@ -803,8 +800,8 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( - M_INV -> uIntListContains(List(tileExclusiveDirty,tileMigratoryDirty),state), - M_CLN -> uIntListContains(List(tileExclusiveDirty,tileMigratoryDirty),state) + M_INV -> Vec(tileExclusiveDirty,tileMigratoryDirty).contains(state), + M_CLN -> Vec(tileExclusiveDirty,tileMigratoryDirty).contains(state) )) } def needsWriteback (state: UInt): Bool = { @@ -875,7 +872,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> Mux(uIntListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), + probeInvalidate -> Mux(Vec(tileExclusiveDirty, tileMigratoryDirty).contains(state), releaseInvalidateDataMigratory, releaseInvalidateData), probeDowngrade -> Mux(state === tileMigratoryDirty, releaseDowngradeDataMigratory, releaseDowngradeData), probeCopy -> releaseCopyData @@ -890,15 +887,13 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uIntListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) + case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } - def messageUpdatesDataArray (reply: Grant): Bool = { - uIntListContains(List(grantReadShared, grantReadExclusive, grantReadMigratory), reply.g_type) - } - def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageUpdatesDataArray (reply: Grant): Bool = Vec(grantReadShared, grantReadExclusive, grantReadMigratory).contains(reply.g_type) + def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) From 39681303b83e275c6bfcf22146e6a98720d1920a Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 22 Apr 2014 16:55:35 -0700 Subject: [PATCH 218/374] beginning of l2 cache --- uncore/src/main/scala/cache.scala | 431 ++++++++++++++++++++++++++++++ 1 file changed, 431 insertions(+) create mode 100644 uncore/src/main/scala/cache.scala diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala new file mode 100644 index 00000000..a4c2755c --- /dev/null +++ b/uncore/src/main/scala/cache.scala @@ -0,0 +1,431 @@ +package uncore +import Chisel._ + +abstract class CacheConfig { + def sets: Int + def ways: Int + def tl: TileLinkConfiguration + def as: AddressSpaceConfiguration + def dm: Boolean + def states: Int + def lines: Int + def tagbits: Int + def idxbits: Int + def offbits: Int + def untagbits: Int + def rowbits: Int + def metabits: Int + def statebits: Int +} + +case class L2CacheConfig(val sets: Int, val ways: Int, + nReleaseTransactions: Int, nAcquireTransactions: Int, + nrpq: Int, nsdq: Int, + val tl: TileLinkConfiguration, + val as: AddressSpaceConfiguration) extends CacheConfig { + def databits = tl.dataBits + def states = tl.co.nMasterStates + def lines = sets*ways + def dm = ways == 1 + def offbits = 0 + def lineaddrbits = tl.addrBits + def idxbits = log2Up(sets) + def waybits = log2Up(ways) + def untagbits = offbits + idxbits + def tagbits = lineaddrbits - idxbits + def databytes = databits/8 + def wordoffbits = log2Up(databytes) + def rowbits = tl.dataBits + def rowbytes = rowbits/8 + def rowoffbits = log2Up(rowbytes) + def refillcycles = tl.dataBits/(rowbits) + def statebits = log2Up(states) + def metabits = statebits + tagbits + + require(states > 0) + require(isPow2(sets)) + require(isPow2(ways)) // TODO: relax this + require(rowbits == tl.dataBits) //TODO: relax this? +} + +abstract trait CacheBundle extends Bundle { + implicit val conf: CacheConfig + override def clone = this.getClass.getConstructors.head.newInstance(conf).asInstanceOf[this.type] +} + +abstract trait L2CacheBundle extends Bundle { + implicit val conf: L2CacheConfig + override def clone = this.getClass.getConstructors.head.newInstance(conf).asInstanceOf[this.type] +} + +abstract class ReplacementPolicy { + def way: UInt + def miss: Unit + def hit: Unit +} + +class RandomReplacement(implicit val conf: CacheConfig) extends ReplacementPolicy { + private val replace = Bool() + replace := Bool(false) + val lfsr = LFSR16(replace) + + def way = if(conf.dm) UInt(0) else lfsr(log2Up(conf.ways)-1,0) + def miss = replace := Bool(true) + def hit = {} +} + +object L2MetaData { + def apply(tag: Bits, state: UInt)(implicit conf: CacheConfig) = { + val meta = new L2MetaData + meta.state := state + meta.tag := tag + meta + } +} +class L2MetaData(implicit val conf: CacheConfig) extends CacheBundle { + val state = UInt(width = conf.statebits) + val tag = Bits(width = conf.tagbits) +} + +class L2MetaReadReq(implicit val conf: CacheConfig) extends CacheBundle { + val idx = Bits(width = conf.idxbits) +} + +class L2MetaWriteReq(implicit conf: CacheConfig) extends L2MetaReadReq()(conf) { + val way_en = Bits(width = conf.ways) + val data = new L2MetaData() +} + +class L2MetaDataArray(implicit conf: CacheConfig) extends Module { + implicit val tl = conf.tl + val io = new Bundle { + val read = Decoupled(new L2MetaReadReq).flip + val write = Decoupled(new L2MetaWriteReq).flip + val resp = Vec.fill(conf.ways){(new L2MetaData).asOutput} + } + + val rst_cnt = Reg(init=UInt(0, log2Up(conf.sets+1))) + val rst = rst_cnt < UInt(conf.sets) + when (rst) { rst_cnt := rst_cnt+UInt(1) } + + val tags = Mem(UInt(width = conf.metabits*conf.ways), conf.sets, seqRead = true) + + when (rst || io.write.valid) { + val addr = Mux(rst, rst_cnt, io.write.bits.idx) + val data = Cat(Mux(rst, tl.co.newStateOnFlush, io.write.bits.data.state), io.write.bits.data.tag) + val mask = Mux(rst, SInt(-1), io.write.bits.way_en) + tags.write(addr, Fill(conf.ways, data), FillInterleaved(conf.metabits, mask)) + } + val tag = tags(RegEnable(io.read.bits.idx, io.read.valid)) + + for (w <- 0 until conf.ways) { + val m = tag(conf.metabits*(w+1)-1, conf.metabits*w) + io.resp(w).state := m >> UInt(conf.tagbits) + io.resp(w).tag := m + } + + io.read.ready := !rst && !io.write.valid // so really this could be a 6T RAM + io.write.ready := !rst +} + +class L2DataReadReq(implicit val conf: L2CacheConfig) extends L2CacheBundle { + val way_en = Bits(width = conf.ways) + val addr = Bits(width = conf.untagbits) +} + +class L2DataWriteReq(implicit conf: L2CacheConfig) extends L2DataReadReq()(conf) { + val wmask = Bits(width = conf.tl.writeMaskBits) + val data = Bits(width = conf.rowbits) +} + +class L2DataArray(implicit conf: L2CacheConfig) extends Module { + val io = new Bundle { + val read = Decoupled(new L2DataReadReq).flip + val write = Decoupled(new L2DataWriteReq).flip + val resp = Vec.fill(conf.ways){Bits(OUTPUT, conf.rowbits)} + } + + val waddr = io.write.bits.addr >> UInt(conf.rowoffbits) + val raddr = io.read.bits.addr >> UInt(conf.rowoffbits) + + val wmask = FillInterleaved(conf.databits, io.write.bits.wmask) + for (w <- 0 until conf.ways) { + val array = Mem(Bits(width=conf.rowbits), conf.sets*conf.refillcycles, seqRead = true) + when (io.write.bits.way_en(w) && io.write.valid) { + array.write(waddr, io.write.bits.data, wmask) + } + io.resp(w) := array(RegEnable(raddr, io.read.bits.way_en(w) && io.read.valid)) + } + + io.read.ready := Bool(true) + io.write.ready := Bool(true) +} + +class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceAgent()(conf.tl) +{ + implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) + + // Create SHRs for outstanding transactions + val nTrackers = conf.nReleaseTransactions + conf.nAcquireTransactions + val trackerList = (0 until conf.nReleaseTransactions).map(id => Module(new L2VoluntaryReleaseTracker(id, bankId))) ++ + (conf.nReleaseTransactions until nTrackers).map(id => Module(new L2AcquireTracker(id, bankId))) + + // Propagate incoherence flags + trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) + + // Handle acquire transaction initiation + val acquire = io.client.acquire + val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) + val block_acquires = any_acquire_conflict + + val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) + for( i <- 0 until trackerList.size ) { + val t = trackerList(i).io.client + alloc_arb.io.in(i).valid := t.acquire.ready + t.acquire.bits := acquire.bits + t.acquire.valid := alloc_arb.io.in(i).ready + } + acquire.ready := trackerList.map(_.io.client.acquire.ready).reduce(_||_) && !block_acquires + alloc_arb.io.out.ready := acquire.valid && !block_acquires + + // Handle probe request generation + val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) + io.client.probe <> probe_arb.io.out + probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.probe } + + // Handle releases, which might be voluntary and might have data + val release = io.client.release + val voluntary = co.isVoluntary(release.bits.payload) + val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) + val block_releases = Bool(false) + val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b} + //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response + val release_idx = Mux(voluntary, UInt(0), release.bits.payload.master_xact_id) + for( i <- 0 until trackerList.size ) { + val t = trackerList(i).io.client + t.release.bits := release.bits + t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases + } + release.ready := Vec(trackerList.map(_.io.client.release.ready)).read(release_idx) && !block_releases + + // Reply to initial requestor + val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) + io.client.grant <> grant_arb.io.out + grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.grant } + + // Free finished transactions + val ack = io.client.grant_ack + trackerList.map(_.io.client.grant_ack.valid := ack.valid) + trackerList.map(_.io.client.grant_ack.bits := ack.bits) + ack.ready := Bool(true) + + // Create an arbiter for the one memory port + val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size)) + outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.master } + io.master <> outer_arb.io.out +} + + +abstract class L2XactTracker()(implicit conf: L2CacheConfig) extends Module { + implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) + val io = new Bundle { + val client = (new TileLinkIO).flip + val master = new UncachedTileLinkIO + val tile_incoherent = Bits(INPUT, ln.nClients) + val has_acquire_conflict = Bool(OUTPUT) + val has_release_conflict = Bool(OUTPUT) + } + + val c_acq = io.client.acquire.bits + val c_rel = io.client.release.bits + val c_gnt = io.client.grant.bits + val c_ack = io.client.grant_ack.bits + val m_gnt = io.master.grant.bits + +} + +class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig) extends L2XactTracker()(conf) { + val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) + val state = Reg(init=s_idle) + val xact = Reg{ new Release } + val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val incoming_rel = io.client.release.bits + + io.has_acquire_conflict := Bool(false) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, incoming_rel.payload.addr) && + (state != s_idle) + + io.master.grant.ready := Bool(false) + io.master.acquire.valid := Bool(false) + io.master.acquire.bits.header.src := UInt(bankId) + //io.master.acquire.bits.header.dst TODO + io.master.acquire.bits.payload := Acquire(co.getUncachedWriteAcquireType, + xact.addr, + UInt(trackerId), + xact.data) + io.client.acquire.ready := Bool(false) + io.client.probe.valid := Bool(false) + io.client.release.ready := Bool(false) + io.client.grant.valid := Bool(false) + io.client.grant.bits.header.src := UInt(bankId) + io.client.grant.bits.header.dst := init_client_id + io.client.grant.bits.payload := Grant(co.getGrantType(xact, UInt(0)), + xact.client_xact_id, + UInt(trackerId)) + + switch (state) { + is(s_idle) { + io.client.release.ready := Bool(true) + when( io.client.release.valid ) { + xact := incoming_rel.payload + init_client_id := incoming_rel.header.src + state := Mux(co.messageHasData(incoming_rel.payload), s_mem, s_ack) + } + } + is(s_mem) { + io.master.acquire.valid := Bool(true) + when(io.master.acquire.ready) { state := s_ack } + } + is(s_ack) { + io.client.grant.valid := Bool(true) + when(io.client.grant.ready) { state := s_idle } + } + } +} + +class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig) extends L2XactTracker()(conf) { + val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) + val state = Reg(init=s_idle) + val xact = Reg{ new Acquire } + val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) + //TODO: Will need id reg for merged release xacts + + val init_sharer_cnt = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val release_count = if (ln.nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(ln.nClients))) + val probe_flags = Reg(init=Bits(0, width = ln.nClients)) + val curr_p_id = PriorityEncoder(probe_flags) + + val pending_outer_write = co.messageHasData(xact) + val pending_outer_read = co.needsOuterRead(xact.a_type, UInt(0)) + val outer_write_acq = Acquire(co.getUncachedWriteAcquireType, + xact.addr, UInt(trackerId), xact.data) + val outer_write_rel = Acquire(co.getUncachedWriteAcquireType, + xact.addr, UInt(trackerId), c_rel.payload.data) + val outer_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) + + val probe_initial_flags = Bits(width = ln.nClients) + probe_initial_flags := Bits(0) + if (ln.nClients > 1) { + // issue self-probes for uncached read xacts to facilitate I$ coherence + val probe_self = Bool(true) //co.needsSelfProbe(io.client.acquire.bits.payload) + val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(ln.nClients)-1,0))) + probe_initial_flags := ~(io.tile_incoherent | myflag) + } + + io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) + + io.master.acquire.valid := Bool(false) + io.master.acquire.bits.header.src := UInt(bankId) + //io.master.acquire.bits.header.dst TODO + io.master.acquire.bits.payload := outer_read + io.master.grant.ready := io.client.grant.ready + + io.client.probe.valid := Bool(false) + io.client.probe.bits.header.src := UInt(bankId) + io.client.probe.bits.header.dst := curr_p_id + io.client.probe.bits.payload := Probe(co.getProbeType(xact.a_type, UInt(0)), + xact.addr, + UInt(trackerId)) + + val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt) + io.client.grant.valid := Bool(false) + io.client.grant.bits.header.src := UInt(bankId) + io.client.grant.bits.header.dst := init_client_id + io.client.grant.bits.payload := Grant(grant_type, + xact.client_xact_id, + UInt(trackerId), + m_gnt.payload.data) + + io.client.acquire.ready := Bool(false) + io.client.release.ready := Bool(false) + + switch (state) { + is(s_idle) { + io.client.acquire.ready := Bool(true) + val needs_outer_write = co.messageHasData(c_acq.payload) + val needs_outer_read = co.needsOuterRead(c_acq.payload.a_type, UInt(0)) + when( io.client.acquire.valid ) { + xact := c_acq.payload + init_client_id := c_acq.header.src + init_sharer_cnt := UInt(ln.nClients) // TODO: Broadcast only + probe_flags := probe_initial_flags + if(ln.nClients > 1) { + release_count := PopCount(probe_initial_flags) + state := Mux(probe_initial_flags.orR, s_probe, + Mux(needs_outer_write, s_mem_write, + Mux(needs_outer_read, s_mem_read, s_make_grant))) + } else state := Mux(needs_outer_write, s_mem_write, + Mux(needs_outer_read, s_mem_read, s_make_grant)) + } + } + is(s_probe) { + // Generate probes + io.client.probe.valid := probe_flags.orR + when(io.client.probe.ready) { + probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) + } + + // Handle releases, which may have data to be written back + when(io.client.release.valid) { + when(co.messageHasData(c_rel.payload)) { + io.master.acquire.valid := Bool(true) + io.master.acquire.bits.payload := outer_write_rel + when(io.master.acquire.ready) { + io.client.release.ready := Bool(true) + if(ln.nClients > 1) release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } + } + } .otherwise { + io.client.release.ready := Bool(true) + if(ln.nClients > 1) release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } + } + } + } + is(s_mem_read) { + io.master.acquire.valid := Bool(true) + io.master.acquire.bits.payload := outer_read + when(io.master.acquire.ready) { + state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) + } + } + is(s_mem_write) { + io.master.acquire.valid := Bool(true) + io.master.acquire.bits.payload := outer_write_acq + when(io.master.acquire.ready) { + state := Mux(pending_outer_read, s_mem_read, s_make_grant) + } + } + is(s_make_grant) { + io.client.grant.valid := Bool(true) + when(io.client.grant.ready) { + state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) + } + } + is(s_busy) { // Nothing left to do but wait for transaction to complete + when(io.master.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { + io.client.grant.valid := Bool(true) + } + when(io.client.grant_ack.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { + state := s_idle + } + } + } +} From f8f29c69b8ad5c619cbe849c69f0eda690ba70dc Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 23 Apr 2014 16:24:20 -0700 Subject: [PATCH 219/374] MetaData & friends moved to uncore/ --- uncore/src/main/scala/cache.scala | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index a4c2755c..bd677c08 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -74,34 +74,34 @@ class RandomReplacement(implicit val conf: CacheConfig) extends ReplacementPolic def hit = {} } -object L2MetaData { +object MetaData { def apply(tag: Bits, state: UInt)(implicit conf: CacheConfig) = { - val meta = new L2MetaData + val meta = new MetaData meta.state := state meta.tag := tag meta } } -class L2MetaData(implicit val conf: CacheConfig) extends CacheBundle { +class MetaData(implicit val conf: CacheConfig) extends CacheBundle { val state = UInt(width = conf.statebits) val tag = Bits(width = conf.tagbits) } -class L2MetaReadReq(implicit val conf: CacheConfig) extends CacheBundle { +class MetaReadReq(implicit val conf: CacheConfig) extends CacheBundle { val idx = Bits(width = conf.idxbits) } -class L2MetaWriteReq(implicit conf: CacheConfig) extends L2MetaReadReq()(conf) { +class MetaWriteReq(implicit conf: CacheConfig) extends MetaReadReq()(conf) { val way_en = Bits(width = conf.ways) - val data = new L2MetaData() + val data = new MetaData() } -class L2MetaDataArray(implicit conf: CacheConfig) extends Module { +class MetaDataArray(implicit conf: CacheConfig) extends Module { implicit val tl = conf.tl val io = new Bundle { - val read = Decoupled(new L2MetaReadReq).flip - val write = Decoupled(new L2MetaWriteReq).flip - val resp = Vec.fill(conf.ways){(new L2MetaData).asOutput} + val read = Decoupled(new MetaReadReq).flip + val write = Decoupled(new MetaWriteReq).flip + val resp = Vec.fill(conf.ways){(new MetaData).asOutput} } val rst_cnt = Reg(init=UInt(0, log2Up(conf.sets+1))) From 3f53d532c25c2039735a3ee2f86f263ce54b4eb1 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sat, 26 Apr 2014 14:58:38 -0700 Subject: [PATCH 220/374] uniquify tilelink conf val name for easier subtyping --- uncore/src/main/scala/tilelink.scala | 36 ++++++++++++++-------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 451c5844..22941d29 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -4,24 +4,24 @@ import Chisel._ case class TileLinkConfiguration(co: CoherencePolicyWithUncached, ln: LogicalNetworkConfiguration, addrBits: Int, masterXactIdBits: Int, clientXactIdBits: Int, dataBits: Int, writeMaskBits: Int, wordAddrBits: Int, atomicOpBits: Int) abstract trait TileLinkSubBundle extends Bundle { - implicit val conf: TileLinkConfiguration - override def clone = this.getClass.getConstructors.head.newInstance(conf).asInstanceOf[this.type] + implicit val tlconf: TileLinkConfiguration + override def clone = this.getClass.getConstructors.head.newInstance(tlconf).asInstanceOf[this.type] } trait HasPhysicalAddress extends TileLinkSubBundle { - val addr = UInt(width = conf.addrBits) + val addr = UInt(width = tlconf.addrBits) } trait HasClientTransactionId extends TileLinkSubBundle { - val client_xact_id = Bits(width = conf.clientXactIdBits) + val client_xact_id = Bits(width = tlconf.clientXactIdBits) } trait HasMasterTransactionId extends TileLinkSubBundle { - val master_xact_id = Bits(width = conf.masterXactIdBits) + val master_xact_id = Bits(width = tlconf.masterXactIdBits) } trait HasTileLinkData extends TileLinkSubBundle { - val data = Bits(width = conf.dataBits) + val data = Bits(width = tlconf.dataBits) } trait SourcedMessage extends Bundle @@ -64,14 +64,14 @@ object Acquire } } -class Acquire(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage +class Acquire(implicit val tlconf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasTileLinkData { - val a_type = UInt(width = conf.co.acquireTypeWidth) - val write_mask = Bits(width = conf.writeMaskBits) - val subword_addr = Bits(width = conf.wordAddrBits) - val atomic_opcode = Bits(width = conf.atomicOpBits) + val a_type = UInt(width = tlconf.co.acquireTypeWidth) + val write_mask = Bits(width = tlconf.writeMaskBits) + val subword_addr = Bits(width = tlconf.wordAddrBits) + val atomic_opcode = Bits(width = tlconf.atomicOpBits) } object Probe @@ -85,10 +85,10 @@ object Probe } } -class Probe(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage +class Probe(implicit val tlconf: TileLinkConfiguration) extends MasterSourcedMessage with HasPhysicalAddress with HasMasterTransactionId { - val p_type = UInt(width = conf.co.probeTypeWidth) + val p_type = UInt(width = tlconf.co.probeTypeWidth) } object Release @@ -116,12 +116,12 @@ object Release } } -class Release(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage +class Release(implicit val tlconf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasMasterTransactionId with HasTileLinkData { - val r_type = UInt(width = conf.co.releaseTypeWidth) + val r_type = UInt(width = tlconf.co.releaseTypeWidth) } object Grant @@ -141,14 +141,14 @@ object Grant } } -class Grant(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage +class Grant(implicit val tlconf: TileLinkConfiguration) extends MasterSourcedMessage with HasTileLinkData with HasClientTransactionId with HasMasterTransactionId { - val g_type = UInt(width = conf.co.grantTypeWidth) + val g_type = UInt(width = tlconf.co.grantTypeWidth) } -class GrantAck(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasMasterTransactionId +class GrantAck(implicit val tlconf: TileLinkConfiguration) extends ClientSourcedMessage with HasMasterTransactionId class UncachedTileLinkIO(implicit conf: TileLinkConfiguration) extends Bundle { From 1163131d1e44804e065a8edea3065111745f535f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sat, 26 Apr 2014 15:17:05 -0700 Subject: [PATCH 221/374] TileLinkIO.GrantAck -> TileLinkIO.Finish --- uncore/src/main/scala/cache.scala | 10 +++++----- uncore/src/main/scala/htif.scala | 8 ++++---- uncore/src/main/scala/tilelink.scala | 16 ++++++++-------- uncore/src/main/scala/uncore.scala | 10 +++++----- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index bd677c08..3af6f29f 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -214,9 +214,9 @@ class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceA grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.grant } // Free finished transactions - val ack = io.client.grant_ack - trackerList.map(_.io.client.grant_ack.valid := ack.valid) - trackerList.map(_.io.client.grant_ack.bits := ack.bits) + val ack = io.client.finish + trackerList.map(_.io.client.finish.valid := ack.valid) + trackerList.map(_.io.client.finish.bits := ack.bits) ack.ready := Bool(true) // Create an arbiter for the one memory port @@ -239,7 +239,7 @@ abstract class L2XactTracker()(implicit conf: L2CacheConfig) extends Module { val c_acq = io.client.acquire.bits val c_rel = io.client.release.bits val c_gnt = io.client.grant.bits - val c_ack = io.client.grant_ack.bits + val c_ack = io.client.finish.bits val m_gnt = io.master.grant.bits } @@ -423,7 +423,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig when(io.master.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { io.client.grant.valid := Bool(true) } - when(io.client.grant_ack.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { + when(io.client.finish.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { state := s_idle } } diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 9ea26f86..41d6df28 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -154,7 +154,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti } mem_acked := Bool(false) } - when (state === state_mem_finish && io.mem.grant_ack.ready) { + when (state === state_mem_finish && io.mem.finish.ready) { state := Mux(cmd === cmd_readmem || pos === UInt(1), state_tx, state_rx) pos := pos - UInt(1) addr := addr + UInt(1 << offsetBits-3) @@ -186,9 +186,9 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti io.mem.acquire.bits.payload.data := mem_req_data io.mem.acquire.bits.header.src := UInt(conf.ln.nClients) // By convention HTIF is the client with the largest id io.mem.acquire.bits.header.dst := UInt(0) // DNC; Overwritten outside module - io.mem.grant_ack.valid := (state === state_mem_finish) && mem_needs_ack - io.mem.grant_ack.bits.payload.master_xact_id := mem_gxid - io.mem.grant_ack.bits.header.dst := mem_gsrc + io.mem.finish.valid := (state === state_mem_finish) && mem_needs_ack + io.mem.finish.bits.payload.master_xact_id := mem_gxid + io.mem.finish.bits.header.dst := mem_gsrc io.mem.probe.ready := Bool(false) io.mem.release.valid := Bool(false) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 22941d29..68da8213 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -148,14 +148,14 @@ class Grant(implicit val tlconf: TileLinkConfiguration) extends MasterSourcedMes val g_type = UInt(width = tlconf.co.grantTypeWidth) } -class GrantAck(implicit val tlconf: TileLinkConfiguration) extends ClientSourcedMessage with HasMasterTransactionId +class Finish(implicit val tlconf: TileLinkConfiguration) extends ClientSourcedMessage with HasMasterTransactionId class UncachedTileLinkIO(implicit conf: TileLinkConfiguration) extends Bundle { implicit val ln = conf.ln val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip - val grant_ack = new DecoupledIO(new LogicalNetworkIO(new GrantAck)) + val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) override def clone = { new UncachedTileLinkIO().asInstanceOf[this.type] } } @@ -214,9 +214,9 @@ abstract class UncachedTileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfigur hookupClientSource(io.in.map(_.acquire), io.out.acquire) hookupMasterSource(io.in.map(_.grant), io.out.grant) - val grant_ack_arb = Module(new RRArbiter(new LogicalNetworkIO(new GrantAck), n)) - io.out.grant_ack <> grant_ack_arb.io.out - grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } + val finish_arb = Module(new RRArbiter(new LogicalNetworkIO(new Finish), n)) + io.out.finish <> finish_arb.io.out + finish_arb.io.in zip io.in map { case (arb, req) => arb <> req.finish } } abstract class TileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkArbiterLike(n)(conf) { @@ -233,9 +233,9 @@ abstract class TileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfiguration) e io.in.map{ _.probe.bits := io.out.probe.bits } io.out.probe.ready := io.in.map(_.probe.ready).reduce(_||_) - val grant_ack_arb = Module(new RRArbiter(new LogicalNetworkIO(new GrantAck), n)) - io.out.grant_ack <> grant_ack_arb.io.out - grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } + val finish_arb = Module(new RRArbiter(new LogicalNetworkIO(new Finish), n)) + io.out.finish <> finish_arb.io.out + finish_arb.io.in zip io.in map { case (arb, req) => arb <> req.finish } } abstract trait AppendsArbiterId { diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 3e162583..0a3b7d6e 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -64,9 +64,9 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.grant } // Free finished transactions - val ack = io.client.grant_ack - trackerList.map(_.io.client.grant_ack.valid := ack.valid) - trackerList.map(_.io.client.grant_ack.bits := ack.bits) + val ack = io.client.finish + trackerList.map(_.io.client.finish.valid := ack.valid) + trackerList.map(_.io.client.finish.bits := ack.bits) ack.ready := Bool(true) // Create an arbiter for the one memory port @@ -89,7 +89,7 @@ abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) exten val c_acq = io.client.acquire.bits val c_rel = io.client.release.bits val c_gnt = io.client.grant.bits - val c_ack = io.client.grant_ack.bits + val c_ack = io.client.finish.bits val m_gnt = io.master.grant.bits } @@ -273,7 +273,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgen when(io.master.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { io.client.grant.valid := Bool(true) } - when(io.client.grant_ack.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { + when(io.client.finish.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { state := s_idle } } From 52c6de56414b9da67957deb258d7bc0bcb3f41c5 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sat, 26 Apr 2014 19:11:36 -0700 Subject: [PATCH 222/374] DRAMSideLLCLike trait. TSHRFile. New L2 config objects. --- uncore/src/main/scala/cache.scala | 114 ++++++++++++++++++++--------- uncore/src/main/scala/llc.scala | 14 ++-- uncore/src/main/scala/uncore.scala | 31 +++++--- 3 files changed, 106 insertions(+), 53 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 3af6f29f..3d42e97d 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -1,7 +1,7 @@ package uncore import Chisel._ -abstract class CacheConfig { +trait CacheConfig { def sets: Int def ways: Int def tl: TileLinkConfiguration @@ -18,12 +18,16 @@ abstract class CacheConfig { def statebits: Int } -case class L2CacheConfig(val sets: Int, val ways: Int, - nReleaseTransactions: Int, nAcquireTransactions: Int, - nrpq: Int, nsdq: Int, - val tl: TileLinkConfiguration, - val as: AddressSpaceConfiguration) extends CacheConfig { - def databits = tl.dataBits +case class L2CacheConfig( + val sets: Int, val ways: Int, + val nrpq: Int, val nsdq: Int, + val nReleaseTransactions: Int, + val nAcquireTransactions: Int, + val tl: TileLinkConfiguration, + val as: AddressSpaceConfiguration) + extends CoherenceAgentConfiguration + with CacheConfig +{ def states = tl.co.nMasterStates def lines = sets*ways def dm = ways == 1 @@ -33,8 +37,9 @@ case class L2CacheConfig(val sets: Int, val ways: Int, def waybits = log2Up(ways) def untagbits = offbits + idxbits def tagbits = lineaddrbits - idxbits - def databytes = databits/8 - def wordoffbits = log2Up(databytes) + def wordbits = 64 + def wordbytes = wordbits/8 + def wordoffbits = log2Up(wordbytes) def rowbits = tl.dataBits def rowbytes = rowbits/8 def rowoffbits = log2Up(rowbytes) @@ -45,17 +50,17 @@ case class L2CacheConfig(val sets: Int, val ways: Int, require(states > 0) require(isPow2(sets)) require(isPow2(ways)) // TODO: relax this - require(rowbits == tl.dataBits) //TODO: relax this? + require(refillcycles == 1) //TODO: relax this? } abstract trait CacheBundle extends Bundle { - implicit val conf: CacheConfig - override def clone = this.getClass.getConstructors.head.newInstance(conf).asInstanceOf[this.type] + implicit val cacheconf: CacheConfig + override def clone = this.getClass.getConstructors.head.newInstance(cacheconf).asInstanceOf[this.type] } abstract trait L2CacheBundle extends Bundle { - implicit val conf: L2CacheConfig - override def clone = this.getClass.getConstructors.head.newInstance(conf).asInstanceOf[this.type] + implicit val l2cacheconf: L2CacheConfig + override def clone = this.getClass.getConstructors.head.newInstance(l2cacheconf).asInstanceOf[this.type] } abstract class ReplacementPolicy { @@ -64,31 +69,31 @@ abstract class ReplacementPolicy { def hit: Unit } -class RandomReplacement(implicit val conf: CacheConfig) extends ReplacementPolicy { +class RandomReplacement(implicit val cacheconf: CacheConfig) extends ReplacementPolicy { private val replace = Bool() replace := Bool(false) val lfsr = LFSR16(replace) - def way = if(conf.dm) UInt(0) else lfsr(log2Up(conf.ways)-1,0) + def way = if(cacheconf.dm) UInt(0) else lfsr(log2Up(cacheconf.ways)-1,0) def miss = replace := Bool(true) def hit = {} } object MetaData { - def apply(tag: Bits, state: UInt)(implicit conf: CacheConfig) = { + def apply(tag: Bits, state: UInt)(implicit cacheconf: CacheConfig) = { val meta = new MetaData meta.state := state meta.tag := tag meta } } -class MetaData(implicit val conf: CacheConfig) extends CacheBundle { - val state = UInt(width = conf.statebits) - val tag = Bits(width = conf.tagbits) +class MetaData(implicit val cacheconf: CacheConfig) extends CacheBundle { + val state = UInt(width = cacheconf.statebits) + val tag = Bits(width = cacheconf.tagbits) } -class MetaReadReq(implicit val conf: CacheConfig) extends CacheBundle { - val idx = Bits(width = conf.idxbits) +class MetaReadReq(implicit val cacheconf: CacheConfig) extends CacheBundle { + val idx = Bits(width = cacheconf.idxbits) } class MetaWriteReq(implicit conf: CacheConfig) extends MetaReadReq()(conf) { @@ -128,27 +133,26 @@ class MetaDataArray(implicit conf: CacheConfig) extends Module { io.write.ready := !rst } -class L2DataReadReq(implicit val conf: L2CacheConfig) extends L2CacheBundle { - val way_en = Bits(width = conf.ways) - val addr = Bits(width = conf.untagbits) +class L2DataReadReq(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle { + val way_en = Bits(width = l2cacheconf.ways) + val addr = Bits(width = l2cacheconf.tl.addrBits) } class L2DataWriteReq(implicit conf: L2CacheConfig) extends L2DataReadReq()(conf) { val wmask = Bits(width = conf.tl.writeMaskBits) - val data = Bits(width = conf.rowbits) + val data = Bits(width = conf.tl.dataBits) } class L2DataArray(implicit conf: L2CacheConfig) extends Module { val io = new Bundle { val read = Decoupled(new L2DataReadReq).flip val write = Decoupled(new L2DataWriteReq).flip - val resp = Vec.fill(conf.ways){Bits(OUTPUT, conf.rowbits)} + val resp = Vec.fill(conf.ways){Bits(OUTPUT, conf.tl.dataBits)} } - val waddr = io.write.bits.addr >> UInt(conf.rowoffbits) - val raddr = io.read.bits.addr >> UInt(conf.rowoffbits) - - val wmask = FillInterleaved(conf.databits, io.write.bits.wmask) + val waddr = io.write.bits.addr + val raddr = io.read.bits.addr + val wmask = FillInterleaved(conf.wordbits, io.write.bits.wmask) for (w <- 0 until conf.ways) { val array = Mem(Bits(width=conf.rowbits), conf.sets*conf.refillcycles, seqRead = true) when (io.write.bits.way_en(w) && io.write.valid) { @@ -161,11 +165,51 @@ class L2DataArray(implicit conf: L2CacheConfig) extends Module { io.write.ready := Bool(true) } -class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceAgent()(conf.tl) -{ +trait InternalRequestState extends CacheBundle { + val tag_match = Bool() + val old_meta = new MetaData + val way_en = Bits(width = cacheconf.ways) +} + +class InternalAcquire(implicit val cacheconf: CacheConfig) extends Acquire()(cacheconf.tl) + with InternalRequestState + +class InternalRelease(implicit val cacheconf: CacheConfig) extends Release()(cacheconf.tl) + with InternalRequestState + +class InternalTileLinkIO(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle { + implicit val (tl, ln) = (l2cacheconf.tl, l2cacheconf.tl.ln) + val acquire = new DecoupledIO(new LogicalNetworkIO(new InternalAcquire)) + val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip + val release = new DecoupledIO(new LogicalNetworkIO(new InternalRelease)) + val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip + val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) +} + +class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceAgent { implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) - // Create SHRs for outstanding transactions + val tshrfile = Module(new TSHRFile(bankId)) + + io.client <> tshrfile.io.client + io.master <> tshrfile.io.outer + io.incoherent <> tshrfile.io.incoherent +} + + +class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { + implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) + val io = new Bundle { + val client = (new InternalTileLinkIO).flip + val outer = new UncachedTileLinkIO + val incoherent = Vec.fill(ln.nClients){Bool()}.asInput + val meta_read_req = Decoupled(new MetaReadReq) + val meta_write_req = Decoupled(new MetaWriteReq) + val data_read_req = Decoupled(new MetaReadReq) + val data_write_req = Decoupled(new MetaWriteReq) + } + + // Create TSHRs for outstanding transactions val nTrackers = conf.nReleaseTransactions + conf.nAcquireTransactions val trackerList = (0 until conf.nReleaseTransactions).map(id => Module(new L2VoluntaryReleaseTracker(id, bankId))) ++ (conf.nReleaseTransactions until nTrackers).map(id => Module(new L2AcquireTracker(id, bankId))) @@ -222,7 +266,7 @@ class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceA // Create an arbiter for the one memory port val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size)) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.master } - io.master <> outer_arb.io.out + io.outer <> outer_arb.io.out } diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index 1bfa4da7..382a5f96 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -343,13 +343,15 @@ class MemReqArb(n: Int, refill_cycles: Int)(implicit conf: MemoryIFConfiguration when (unlock) { lock := Bool(false) } } -class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, tagLeaf: Mem[UInt], dataLeaf: Mem[UInt])(implicit conf: MemoryIFConfiguration) extends Module -{ +abstract class DRAMSideLLCLike(implicit conf: MemoryIFConfiguration) extends Module { val io = new Bundle { val cpu = new MemIO().flip val mem = new MemPipeIO } +} +class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, tagLeaf: Mem[UInt], dataLeaf: Mem[UInt])(implicit conf: MemoryIFConfiguration) extends DRAMSideLLCLike +{ val tagWidth = conf.addrBits - log2Up(sets) val metaWidth = tagWidth + 2 // valid + dirty @@ -497,12 +499,8 @@ object HellaQueue } } -class DRAMSideLLCNull(numRequests: Int, refillCycles: Int)(implicit conf: MemoryIFConfiguration) extends Module -{ - val io = new Bundle { - val cpu = new MemIO().flip - val mem = new MemPipeIO - } +class DRAMSideLLCNull(numRequests: Int, refillCycles: Int)(implicit conf: MemoryIFConfiguration) + extends DRAMSideLLCLike { val numEntries = numRequests * refillCycles val size = log2Down(numEntries) + 1 diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 0a3b7d6e..cbcd39b6 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -1,18 +1,27 @@ package uncore import Chisel._ -abstract class CoherenceAgent(implicit conf: TileLinkConfiguration) extends Module { +trait CoherenceAgentConfiguration { + def tl: TileLinkConfiguration + def nReleaseTransactions: Int + def nAcquireTransactions: Int +} + +abstract class CoherenceAgent(implicit conf: CoherenceAgentConfiguration) extends Module { val io = new Bundle { - val client = (new TileLinkIO).flip - val master = new UncachedTileLinkIO - val incoherent = Vec.fill(conf.ln.nClients){Bool()}.asInput + val client = (new TileLinkIO()(conf.tl)).flip + val master = new UncachedTileLinkIO()(conf.tl) + val incoherent = Vec.fill(conf.tl.ln.nClients){Bool()}.asInput } } -case class L2CoherenceAgentConfiguration(tl: TileLinkConfiguration, nReleaseTransactions: Int, nAcquireTransactions: Int) +case class L2CoherenceAgentConfiguration( + val tl: TileLinkConfiguration, + val nReleaseTransactions: Int, + val nAcquireTransactions: Int) extends CoherenceAgentConfiguration -class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends CoherenceAgent()(conf.tl) -{ +class L2CoherenceAgent(bankId: Int)(implicit conf: CoherenceAgentConfiguration) + extends CoherenceAgent { implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) // Create SHRs for outstanding transactions @@ -76,7 +85,7 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration } -abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) extends Module { +abstract class XactTracker()(implicit conf: CoherenceAgentConfiguration) extends Module { implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val io = new Bundle { val client = (new TileLinkIO).flip @@ -94,7 +103,8 @@ abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) exten } -class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { +class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentConfiguration) + extends XactTracker()(conf) { val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) val xact = Reg{ new Release } @@ -143,7 +153,8 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Cohe } } -class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { +class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentConfiguration) + extends XactTracker()(conf) { val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) val state = Reg(init=s_idle) val xact = Reg{ new Acquire } From 023722992162c123fee90d2395c14d7c866a2dd6 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 29 Apr 2014 16:49:18 -0700 Subject: [PATCH 223/374] client/master -> inner/outer --- uncore/src/main/scala/cache.scala | 160 ++++++++++++++--------------- uncore/src/main/scala/uncore.scala | 160 ++++++++++++++--------------- 2 files changed, 160 insertions(+), 160 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 3d42e97d..4eb28f97 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -191,8 +191,8 @@ class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceA val tshrfile = Module(new TSHRFile(bankId)) - io.client <> tshrfile.io.client - io.master <> tshrfile.io.outer + tshrfile.io.inner <> io.inner + io.outer <> tshrfile.io.outer io.incoherent <> tshrfile.io.incoherent } @@ -200,7 +200,7 @@ class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceA class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val io = new Bundle { - val client = (new InternalTileLinkIO).flip + val inner = (new InternalTileLinkIO).flip val outer = new UncachedTileLinkIO val incoherent = Vec.fill(ln.nClients){Bool()}.asInput val meta_read_req = Decoupled(new MetaReadReq) @@ -218,27 +218,27 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) // Handle acquire transaction initiation - val acquire = io.client.acquire + val acquire = io.inner.acquire val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val block_acquires = any_acquire_conflict val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.client + val t = trackerList(i).io.inner alloc_arb.io.in(i).valid := t.acquire.ready t.acquire.bits := acquire.bits t.acquire.valid := alloc_arb.io.in(i).ready } - acquire.ready := trackerList.map(_.io.client.acquire.ready).reduce(_||_) && !block_acquires + acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && !block_acquires alloc_arb.io.out.ready := acquire.valid && !block_acquires // Handle probe request generation val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) - io.client.probe <> probe_arb.io.out - probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.probe } + io.inner.probe <> probe_arb.io.out + probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.probe } // Handle releases, which might be voluntary and might have data - val release = io.client.release + val release = io.inner.release val voluntary = co.isVoluntary(release.bits.payload) val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) val block_releases = Bool(false) @@ -246,26 +246,26 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response val release_idx = Mux(voluntary, UInt(0), release.bits.payload.master_xact_id) for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.client + val t = trackerList(i).io.inner t.release.bits := release.bits t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases } - release.ready := Vec(trackerList.map(_.io.client.release.ready)).read(release_idx) && !block_releases + release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases // Reply to initial requestor val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) - io.client.grant <> grant_arb.io.out - grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.grant } + io.inner.grant <> grant_arb.io.out + grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } // Free finished transactions - val ack = io.client.finish - trackerList.map(_.io.client.finish.valid := ack.valid) - trackerList.map(_.io.client.finish.bits := ack.bits) + val ack = io.inner.finish + trackerList.map(_.io.inner.finish.valid := ack.valid) + trackerList.map(_.io.inner.finish.bits := ack.bits) ack.ready := Bool(true) // Create an arbiter for the one memory port val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size)) - outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.master } + outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } io.outer <> outer_arb.io.out } @@ -273,18 +273,18 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { abstract class L2XactTracker()(implicit conf: L2CacheConfig) extends Module { implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val io = new Bundle { - val client = (new TileLinkIO).flip - val master = new UncachedTileLinkIO + val inner = (new InternalTileLinkIO).flip + val outer = new UncachedTileLinkIO val tile_incoherent = Bits(INPUT, ln.nClients) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) } - val c_acq = io.client.acquire.bits - val c_rel = io.client.release.bits - val c_gnt = io.client.grant.bits - val c_ack = io.client.finish.bits - val m_gnt = io.master.grant.bits + val c_acq = io.inner.acquire.bits + val c_rel = io.inner.release.bits + val c_gnt = io.inner.grant.bits + val c_ack = io.inner.finish.bits + val m_gnt = io.outer.grant.bits } @@ -293,46 +293,46 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Ca val state = Reg(init=s_idle) val xact = Reg{ new Release } val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) - val incoming_rel = io.client.release.bits + val incoming_rel = io.inner.release.bits io.has_acquire_conflict := Bool(false) io.has_release_conflict := co.isCoherenceConflict(xact.addr, incoming_rel.payload.addr) && (state != s_idle) - io.master.grant.ready := Bool(false) - io.master.acquire.valid := Bool(false) - io.master.acquire.bits.header.src := UInt(bankId) - //io.master.acquire.bits.header.dst TODO - io.master.acquire.bits.payload := Acquire(co.getUncachedWriteAcquireType, + io.outer.grant.ready := Bool(false) + io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.header.src := UInt(bankId) + //io.outer.acquire.bits.header.dst TODO + io.outer.acquire.bits.payload := Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId), xact.data) - io.client.acquire.ready := Bool(false) - io.client.probe.valid := Bool(false) - io.client.release.ready := Bool(false) - io.client.grant.valid := Bool(false) - io.client.grant.bits.header.src := UInt(bankId) - io.client.grant.bits.header.dst := init_client_id - io.client.grant.bits.payload := Grant(co.getGrantType(xact, UInt(0)), + io.inner.acquire.ready := Bool(false) + io.inner.probe.valid := Bool(false) + io.inner.release.ready := Bool(false) + io.inner.grant.valid := Bool(false) + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := init_client_id + io.inner.grant.bits.payload := Grant(co.getGrantType(xact, UInt(0)), xact.client_xact_id, UInt(trackerId)) switch (state) { is(s_idle) { - io.client.release.ready := Bool(true) - when( io.client.release.valid ) { + io.inner.release.ready := Bool(true) + when( io.inner.release.valid ) { xact := incoming_rel.payload init_client_id := incoming_rel.header.src state := Mux(co.messageHasData(incoming_rel.payload), s_mem, s_ack) } } is(s_mem) { - io.master.acquire.valid := Bool(true) - when(io.master.acquire.ready) { state := s_ack } + io.outer.acquire.valid := Bool(true) + when(io.outer.acquire.ready) { state := s_ack } } is(s_ack) { - io.client.grant.valid := Bool(true) - when(io.client.grant.ready) { state := s_idle } + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { state := s_idle } } } } @@ -361,7 +361,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig probe_initial_flags := Bits(0) if (ln.nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence - val probe_self = Bool(true) //co.needsSelfProbe(io.client.acquire.bits.payload) + val probe_self = Bool(true) //co.needsSelfProbe(io.inner.acquire.bits.payload) val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(ln.nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } @@ -369,37 +369,37 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) - io.master.acquire.valid := Bool(false) - io.master.acquire.bits.header.src := UInt(bankId) - //io.master.acquire.bits.header.dst TODO - io.master.acquire.bits.payload := outer_read - io.master.grant.ready := io.client.grant.ready + io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.header.src := UInt(bankId) + //io.outer.acquire.bits.header.dst TODO + io.outer.acquire.bits.payload := outer_read + io.outer.grant.ready := io.inner.grant.ready - io.client.probe.valid := Bool(false) - io.client.probe.bits.header.src := UInt(bankId) - io.client.probe.bits.header.dst := curr_p_id - io.client.probe.bits.payload := Probe(co.getProbeType(xact.a_type, UInt(0)), + io.inner.probe.valid := Bool(false) + io.inner.probe.bits.header.src := UInt(bankId) + io.inner.probe.bits.header.dst := curr_p_id + io.inner.probe.bits.payload := Probe(co.getProbeType(xact.a_type, UInt(0)), xact.addr, UInt(trackerId)) val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt) - io.client.grant.valid := Bool(false) - io.client.grant.bits.header.src := UInt(bankId) - io.client.grant.bits.header.dst := init_client_id - io.client.grant.bits.payload := Grant(grant_type, + io.inner.grant.valid := Bool(false) + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := init_client_id + io.inner.grant.bits.payload := Grant(grant_type, xact.client_xact_id, UInt(trackerId), m_gnt.payload.data) - io.client.acquire.ready := Bool(false) - io.client.release.ready := Bool(false) + io.inner.acquire.ready := Bool(false) + io.inner.release.ready := Bool(false) switch (state) { is(s_idle) { - io.client.acquire.ready := Bool(true) + io.inner.acquire.ready := Bool(true) val needs_outer_write = co.messageHasData(c_acq.payload) val needs_outer_read = co.needsOuterRead(c_acq.payload.a_type, UInt(0)) - when( io.client.acquire.valid ) { + when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src init_sharer_cnt := UInt(ln.nClients) // TODO: Broadcast only @@ -415,18 +415,18 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig } is(s_probe) { // Generate probes - io.client.probe.valid := probe_flags.orR - when(io.client.probe.ready) { + io.inner.probe.valid := probe_flags.orR + when(io.inner.probe.ready) { probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) } // Handle releases, which may have data to be written back - when(io.client.release.valid) { + when(io.inner.release.valid) { when(co.messageHasData(c_rel.payload)) { - io.master.acquire.valid := Bool(true) - io.master.acquire.bits.payload := outer_write_rel - when(io.master.acquire.ready) { - io.client.release.ready := Bool(true) + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_write_rel + when(io.outer.acquire.ready) { + io.inner.release.ready := Bool(true) if(ln.nClients > 1) release_count := release_count - UInt(1) when(release_count === UInt(1)) { state := Mux(pending_outer_write, s_mem_write, @@ -434,7 +434,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig } } } .otherwise { - io.client.release.ready := Bool(true) + io.inner.release.ready := Bool(true) if(ln.nClients > 1) release_count := release_count - UInt(1) when(release_count === UInt(1)) { state := Mux(pending_outer_write, s_mem_write, @@ -444,30 +444,30 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig } } is(s_mem_read) { - io.master.acquire.valid := Bool(true) - io.master.acquire.bits.payload := outer_read - when(io.master.acquire.ready) { + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_read + when(io.outer.acquire.ready) { state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) } } is(s_mem_write) { - io.master.acquire.valid := Bool(true) - io.master.acquire.bits.payload := outer_write_acq - when(io.master.acquire.ready) { + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_write_acq + when(io.outer.acquire.ready) { state := Mux(pending_outer_read, s_mem_read, s_make_grant) } } is(s_make_grant) { - io.client.grant.valid := Bool(true) - when(io.client.grant.ready) { + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) } } is(s_busy) { // Nothing left to do but wait for transaction to complete - when(io.master.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { - io.client.grant.valid := Bool(true) + when(io.outer.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { + io.inner.grant.valid := Bool(true) } - when(io.client.finish.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { + when(io.inner.finish.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { state := s_idle } } diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index cbcd39b6..c8e33216 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -9,8 +9,8 @@ trait CoherenceAgentConfiguration { abstract class CoherenceAgent(implicit conf: CoherenceAgentConfiguration) extends Module { val io = new Bundle { - val client = (new TileLinkIO()(conf.tl)).flip - val master = new UncachedTileLinkIO()(conf.tl) + val inner = (new TileLinkIO()(conf.tl)).flip + val outer = new UncachedTileLinkIO()(conf.tl) val incoherent = Vec.fill(conf.tl.ln.nClients){Bool()}.asInput } } @@ -33,27 +33,27 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: CoherenceAgentConfiguration) trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) // Handle acquire transaction initiation - val acquire = io.client.acquire + val acquire = io.inner.acquire val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val block_acquires = any_acquire_conflict val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.client + val t = trackerList(i).io.inner alloc_arb.io.in(i).valid := t.acquire.ready t.acquire.bits := acquire.bits t.acquire.valid := alloc_arb.io.in(i).ready } - acquire.ready := trackerList.map(_.io.client.acquire.ready).reduce(_||_) && !block_acquires + acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && !block_acquires alloc_arb.io.out.ready := acquire.valid && !block_acquires // Handle probe request generation val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) - io.client.probe <> probe_arb.io.out - probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.probe } + io.inner.probe <> probe_arb.io.out + probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.probe } // Handle releases, which might be voluntary and might have data - val release = io.client.release + val release = io.inner.release val voluntary = co.isVoluntary(release.bits.payload) val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) val block_releases = Bool(false) @@ -61,45 +61,45 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: CoherenceAgentConfiguration) //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response val release_idx = Mux(voluntary, UInt(0), release.bits.payload.master_xact_id) for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.client + val t = trackerList(i).io.inner t.release.bits := release.bits t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases } - release.ready := Vec(trackerList.map(_.io.client.release.ready)).read(release_idx) && !block_releases + release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases // Reply to initial requestor val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) - io.client.grant <> grant_arb.io.out - grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.grant } + io.inner.grant <> grant_arb.io.out + grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } // Free finished transactions - val ack = io.client.finish - trackerList.map(_.io.client.finish.valid := ack.valid) - trackerList.map(_.io.client.finish.bits := ack.bits) + val ack = io.inner.finish + trackerList.map(_.io.inner.finish.valid := ack.valid) + trackerList.map(_.io.inner.finish.bits := ack.bits) ack.ready := Bool(true) // Create an arbiter for the one memory port val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size)) - outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.master } - io.master <> outer_arb.io.out + outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } + io.outer <> outer_arb.io.out } abstract class XactTracker()(implicit conf: CoherenceAgentConfiguration) extends Module { implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val io = new Bundle { - val client = (new TileLinkIO).flip - val master = new UncachedTileLinkIO + val inner = (new TileLinkIO).flip + val outer = new UncachedTileLinkIO val tile_incoherent = Bits(INPUT, ln.nClients) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) } - val c_acq = io.client.acquire.bits - val c_rel = io.client.release.bits - val c_gnt = io.client.grant.bits - val c_ack = io.client.finish.bits - val m_gnt = io.master.grant.bits + val c_acq = io.inner.acquire.bits + val c_rel = io.inner.release.bits + val c_gnt = io.inner.grant.bits + val c_ack = io.inner.finish.bits + val m_gnt = io.outer.grant.bits } @@ -109,46 +109,46 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: Cohere val state = Reg(init=s_idle) val xact = Reg{ new Release } val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) - val incoming_rel = io.client.release.bits + val incoming_rel = io.inner.release.bits io.has_acquire_conflict := Bool(false) io.has_release_conflict := co.isCoherenceConflict(xact.addr, incoming_rel.payload.addr) && (state != s_idle) - io.master.grant.ready := Bool(false) - io.master.acquire.valid := Bool(false) - io.master.acquire.bits.header.src := UInt(bankId) - //io.master.acquire.bits.header.dst TODO - io.master.acquire.bits.payload := Acquire(co.getUncachedWriteAcquireType, + io.outer.grant.ready := Bool(false) + io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.header.src := UInt(bankId) + //io.outer.acquire.bits.header.dst TODO + io.outer.acquire.bits.payload := Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId), xact.data) - io.client.acquire.ready := Bool(false) - io.client.probe.valid := Bool(false) - io.client.release.ready := Bool(false) - io.client.grant.valid := Bool(false) - io.client.grant.bits.header.src := UInt(bankId) - io.client.grant.bits.header.dst := init_client_id - io.client.grant.bits.payload := Grant(co.getGrantType(xact, UInt(0)), + io.inner.acquire.ready := Bool(false) + io.inner.probe.valid := Bool(false) + io.inner.release.ready := Bool(false) + io.inner.grant.valid := Bool(false) + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := init_client_id + io.inner.grant.bits.payload := Grant(co.getGrantType(xact, UInt(0)), xact.client_xact_id, UInt(trackerId)) switch (state) { is(s_idle) { - io.client.release.ready := Bool(true) - when( io.client.release.valid ) { + io.inner.release.ready := Bool(true) + when( io.inner.release.valid ) { xact := incoming_rel.payload init_client_id := incoming_rel.header.src state := Mux(co.messageHasData(incoming_rel.payload), s_mem, s_ack) } } is(s_mem) { - io.master.acquire.valid := Bool(true) - when(io.master.acquire.ready) { state := s_ack } + io.outer.acquire.valid := Bool(true) + when(io.outer.acquire.ready) { state := s_ack } } is(s_ack) { - io.client.grant.valid := Bool(true) - when(io.client.grant.ready) { state := s_idle } + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { state := s_idle } } } } @@ -178,7 +178,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC probe_initial_flags := Bits(0) if (ln.nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence - val probe_self = Bool(true) //co.needsSelfProbe(io.client.acquire.bits.payload) + val probe_self = Bool(true) //co.needsSelfProbe(io.inner.acquire.bits.payload) val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(ln.nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } @@ -186,37 +186,37 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) - io.master.acquire.valid := Bool(false) - io.master.acquire.bits.header.src := UInt(bankId) - //io.master.acquire.bits.header.dst TODO - io.master.acquire.bits.payload := outer_read - io.master.grant.ready := io.client.grant.ready + io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.header.src := UInt(bankId) + //io.outer.acquire.bits.header.dst TODO + io.outer.acquire.bits.payload := outer_read + io.outer.grant.ready := io.inner.grant.ready - io.client.probe.valid := Bool(false) - io.client.probe.bits.header.src := UInt(bankId) - io.client.probe.bits.header.dst := curr_p_id - io.client.probe.bits.payload := Probe(co.getProbeType(xact.a_type, UInt(0)), + io.inner.probe.valid := Bool(false) + io.inner.probe.bits.header.src := UInt(bankId) + io.inner.probe.bits.header.dst := curr_p_id + io.inner.probe.bits.payload := Probe(co.getProbeType(xact.a_type, UInt(0)), xact.addr, UInt(trackerId)) val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt) - io.client.grant.valid := Bool(false) - io.client.grant.bits.header.src := UInt(bankId) - io.client.grant.bits.header.dst := init_client_id - io.client.grant.bits.payload := Grant(grant_type, + io.inner.grant.valid := Bool(false) + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := init_client_id + io.inner.grant.bits.payload := Grant(grant_type, xact.client_xact_id, UInt(trackerId), m_gnt.payload.data) - io.client.acquire.ready := Bool(false) - io.client.release.ready := Bool(false) + io.inner.acquire.ready := Bool(false) + io.inner.release.ready := Bool(false) switch (state) { is(s_idle) { - io.client.acquire.ready := Bool(true) + io.inner.acquire.ready := Bool(true) val needs_outer_write = co.messageHasData(c_acq.payload) val needs_outer_read = co.needsOuterRead(c_acq.payload.a_type, UInt(0)) - when( io.client.acquire.valid ) { + when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src init_sharer_cnt := UInt(ln.nClients) // TODO: Broadcast only @@ -232,18 +232,18 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC } is(s_probe) { // Generate probes - io.client.probe.valid := probe_flags.orR - when(io.client.probe.ready) { + io.inner.probe.valid := probe_flags.orR + when(io.inner.probe.ready) { probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) } // Handle releases, which may have data to be written back - when(io.client.release.valid) { + when(io.inner.release.valid) { when(co.messageHasData(c_rel.payload)) { - io.master.acquire.valid := Bool(true) - io.master.acquire.bits.payload := outer_write_rel - when(io.master.acquire.ready) { - io.client.release.ready := Bool(true) + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_write_rel + when(io.outer.acquire.ready) { + io.inner.release.ready := Bool(true) if(ln.nClients > 1) release_count := release_count - UInt(1) when(release_count === UInt(1)) { state := Mux(pending_outer_write, s_mem_write, @@ -251,7 +251,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC } } } .otherwise { - io.client.release.ready := Bool(true) + io.inner.release.ready := Bool(true) if(ln.nClients > 1) release_count := release_count - UInt(1) when(release_count === UInt(1)) { state := Mux(pending_outer_write, s_mem_write, @@ -261,30 +261,30 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC } } is(s_mem_read) { - io.master.acquire.valid := Bool(true) - io.master.acquire.bits.payload := outer_read - when(io.master.acquire.ready) { + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_read + when(io.outer.acquire.ready) { state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) } } is(s_mem_write) { - io.master.acquire.valid := Bool(true) - io.master.acquire.bits.payload := outer_write_acq - when(io.master.acquire.ready) { + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_write_acq + when(io.outer.acquire.ready) { state := Mux(pending_outer_read, s_mem_read, s_make_grant) } } is(s_make_grant) { - io.client.grant.valid := Bool(true) - when(io.client.grant.ready) { + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) } } is(s_busy) { // Nothing left to do but wait for transaction to complete - when(io.master.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { - io.client.grant.valid := Bool(true) + when(io.outer.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { + io.inner.grant.valid := Bool(true) } - when(io.client.finish.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { + when(io.inner.finish.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { state := s_idle } } From 45172f1f37eb5e0633bf9eb2cd700d3cd15979f5 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 1 May 2014 01:44:59 -0700 Subject: [PATCH 224/374] parameterize metadataarray --- uncore/src/main/scala/cache.scala | 91 +++++++++++++++++++------------ 1 file changed, 55 insertions(+), 36 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 4eb28f97..610317d2 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -14,8 +14,6 @@ trait CacheConfig { def offbits: Int def untagbits: Int def rowbits: Int - def metabits: Int - def statebits: Int } case class L2CacheConfig( @@ -45,7 +43,6 @@ case class L2CacheConfig( def rowoffbits = log2Up(rowbytes) def refillcycles = tl.dataBits/(rowbits) def statebits = log2Up(states) - def metabits = statebits + tagbits require(states > 0) require(isPow2(sets)) @@ -79,54 +76,61 @@ class RandomReplacement(implicit val cacheconf: CacheConfig) extends Replacement def hit = {} } -object MetaData { - def apply(tag: Bits, state: UInt)(implicit cacheconf: CacheConfig) = { - val meta = new MetaData - meta.state := state - meta.tag := tag - meta - } -} -class MetaData(implicit val cacheconf: CacheConfig) extends CacheBundle { - val state = UInt(width = cacheconf.statebits) +abstract class MetaData(implicit val cacheconf: CacheConfig) extends CacheBundle { val tag = Bits(width = cacheconf.tagbits) } +class L2MetaData(implicit val l2cacheconf: L2CacheConfig) extends MetaData + with L2CacheBundle { + val state = UInt(width = l2cacheconf.statebits) + val sharers = Bits(width = l2cacheconf.tl.ln.nClients) +} + +/* +class L3MetaData(implicit conf: L3CacheConfig) extends MetaData()(conf) { + val cstate = UInt(width = cacheconf.cstatebits) + val mstate = UInt(width = cacheconf.mstatebits) + val sharers = Bits(width = cacheconf.tl.ln.nClients) +} +*/ + class MetaReadReq(implicit val cacheconf: CacheConfig) extends CacheBundle { val idx = Bits(width = cacheconf.idxbits) } -class MetaWriteReq(implicit conf: CacheConfig) extends MetaReadReq()(conf) { +class MetaWriteReq[T <: MetaData](gen: T)(implicit conf: CacheConfig) extends MetaReadReq { val way_en = Bits(width = conf.ways) - val data = new MetaData() + val data = gen.clone + override def clone = new MetaWriteReq(gen)(conf).asInstanceOf[this.type] } -class MetaDataArray(implicit conf: CacheConfig) extends Module { +class MetaDataArray[T <: MetaData](resetMeta: T)(implicit conf: CacheConfig) extends Module { implicit val tl = conf.tl + def gen = resetMeta.clone val io = new Bundle { val read = Decoupled(new MetaReadReq).flip - val write = Decoupled(new MetaWriteReq).flip - val resp = Vec.fill(conf.ways){(new MetaData).asOutput} + val write = Decoupled(new MetaWriteReq(gen)).flip + val resp = Vec.fill(conf.ways){gen.asOutput} } + val metabits = resetMeta.getWidth val rst_cnt = Reg(init=UInt(0, log2Up(conf.sets+1))) val rst = rst_cnt < UInt(conf.sets) when (rst) { rst_cnt := rst_cnt+UInt(1) } - val tags = Mem(UInt(width = conf.metabits*conf.ways), conf.sets, seqRead = true) + val tags = Mem(UInt(width = metabits*conf.ways), conf.sets, seqRead = true) when (rst || io.write.valid) { val addr = Mux(rst, rst_cnt, io.write.bits.idx) - val data = Cat(Mux(rst, tl.co.newStateOnFlush, io.write.bits.data.state), io.write.bits.data.tag) + val data = Mux(rst, resetMeta, io.write.bits.data).toBits val mask = Mux(rst, SInt(-1), io.write.bits.way_en) - tags.write(addr, Fill(conf.ways, data), FillInterleaved(conf.metabits, mask)) + tags.write(addr, Fill(conf.ways, data), FillInterleaved(metabits, mask)) } val tag = tags(RegEnable(io.read.bits.idx, io.read.valid)) for (w <- 0 until conf.ways) { - val m = tag(conf.metabits*(w+1)-1, conf.metabits*w) - io.resp(w).state := m >> UInt(conf.tagbits) - io.resp(w).tag := m + val m = tag(metabits*(w+1)-1, metabits*w) + io.resp(w) := gen.fromBits(m) } io.read.ready := !rst && !io.write.valid // so really this could be a 6T RAM @@ -165,23 +169,23 @@ class L2DataArray(implicit conf: L2CacheConfig) extends Module { io.write.ready := Bool(true) } -trait InternalRequestState extends CacheBundle { +trait L2InternalRequestState extends L2CacheBundle { val tag_match = Bool() - val old_meta = new MetaData - val way_en = Bits(width = cacheconf.ways) + val old_meta = new L2MetaData + val way_en = Bits(width = l2cacheconf.ways) } -class InternalAcquire(implicit val cacheconf: CacheConfig) extends Acquire()(cacheconf.tl) - with InternalRequestState +class L2InternalAcquire(implicit val l2cacheconf: L2CacheConfig) extends Acquire()(l2cacheconf.tl) + with L2InternalRequestState -class InternalRelease(implicit val cacheconf: CacheConfig) extends Release()(cacheconf.tl) - with InternalRequestState +class L2InternalRelease(implicit val l2cacheconf: L2CacheConfig) extends Release()(l2cacheconf.tl) + with L2InternalRequestState class InternalTileLinkIO(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle { implicit val (tl, ln) = (l2cacheconf.tl, l2cacheconf.tl.ln) - val acquire = new DecoupledIO(new LogicalNetworkIO(new InternalAcquire)) + val acquire = new DecoupledIO(new LogicalNetworkIO(new L2InternalAcquire)) val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip - val release = new DecoupledIO(new LogicalNetworkIO(new InternalRelease)) + val release = new DecoupledIO(new LogicalNetworkIO(new L2InternalRelease)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) } @@ -191,6 +195,21 @@ class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceA val tshrfile = Module(new TSHRFile(bankId)) + // tags + val meta = Module(new MetaDataArray(new L2MetaData)) + + // data + val data = Module(new L2DataArray) + + // replacement policy + val replacer = new RandomReplacement +/* + val s1_replaced_way_en = UIntToOH(replacer.way) + val s2_replaced_way_en = UIntToOH(RegEnable(replacer.way, s1_clk_en)) + val s2_repl_meta = Mux1H(s2_replaced_way_en, wayMap((w: Int) => + RegEnable(meta.io.resp(w), s1_clk_en && s1_replaced_way_en(w))).toSeq) +*/ + tshrfile.io.inner <> io.inner io.outer <> tshrfile.io.outer io.incoherent <> tshrfile.io.incoherent @@ -204,9 +223,9 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { val outer = new UncachedTileLinkIO val incoherent = Vec.fill(ln.nClients){Bool()}.asInput val meta_read_req = Decoupled(new MetaReadReq) - val meta_write_req = Decoupled(new MetaWriteReq) - val data_read_req = Decoupled(new MetaReadReq) - val data_write_req = Decoupled(new MetaWriteReq) + val meta_write_req = Decoupled(new MetaWriteReq(new L2MetaData)) + val data_read_req = Decoupled(new L2DataReadReq) + val data_write_req = Decoupled(new L2DataWriteReq) } // Create TSHRs for outstanding transactions From bc3ef1011eb7182d48f739bfff823772e36d2f62 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 6 May 2014 12:59:45 -0700 Subject: [PATCH 225/374] correct use of function value to initialize MetaDataArray --- uncore/src/main/scala/cache.scala | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 610317d2..e41e1cfb 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -104,16 +104,16 @@ class MetaWriteReq[T <: MetaData](gen: T)(implicit conf: CacheConfig) extends Me override def clone = new MetaWriteReq(gen)(conf).asInstanceOf[this.type] } -class MetaDataArray[T <: MetaData](resetMeta: T)(implicit conf: CacheConfig) extends Module { +class MetaDataArray[T <: MetaData](gen: () => T)(implicit conf: CacheConfig) extends Module { implicit val tl = conf.tl - def gen = resetMeta.clone + val rstVal = gen() val io = new Bundle { val read = Decoupled(new MetaReadReq).flip - val write = Decoupled(new MetaWriteReq(gen)).flip - val resp = Vec.fill(conf.ways){gen.asOutput} + val write = Decoupled(new MetaWriteReq(rstVal.clone)).flip + val resp = Vec.fill(conf.ways){rstVal.clone.asOutput} } - val metabits = resetMeta.getWidth + val metabits = rstVal.getWidth val rst_cnt = Reg(init=UInt(0, log2Up(conf.sets+1))) val rst = rst_cnt < UInt(conf.sets) when (rst) { rst_cnt := rst_cnt+UInt(1) } @@ -122,7 +122,7 @@ class MetaDataArray[T <: MetaData](resetMeta: T)(implicit conf: CacheConfig) ext when (rst || io.write.valid) { val addr = Mux(rst, rst_cnt, io.write.bits.idx) - val data = Mux(rst, resetMeta, io.write.bits.data).toBits + val data = Mux(rst, rstVal, io.write.bits.data).toBits val mask = Mux(rst, SInt(-1), io.write.bits.way_en) tags.write(addr, Fill(conf.ways, data), FillInterleaved(metabits, mask)) } @@ -130,7 +130,7 @@ class MetaDataArray[T <: MetaData](resetMeta: T)(implicit conf: CacheConfig) ext for (w <- 0 until conf.ways) { val m = tag(metabits*(w+1)-1, metabits*w) - io.resp(w) := gen.fromBits(m) + io.resp(w) := rstVal.clone.fromBits(m) } io.read.ready := !rst && !io.write.valid // so really this could be a 6T RAM @@ -196,7 +196,7 @@ class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceA val tshrfile = Module(new TSHRFile(bankId)) // tags - val meta = Module(new MetaDataArray(new L2MetaData)) + val meta = Module(new MetaDataArray(() => new L2MetaData)) // data val data = Module(new L2DataArray) From 0e39346a126778862361b8adf43834f14e0fd3fd Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 7 May 2014 01:51:46 -0700 Subject: [PATCH 226/374] L2-specific metadataarray wrapper, hookups to tshrfile --- uncore/src/main/scala/cache.scala | 212 +++++++++++++++++++++--------- 1 file changed, 153 insertions(+), 59 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index e41e1cfb..e5851a6b 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -43,6 +43,7 @@ case class L2CacheConfig( def rowoffbits = log2Up(rowbytes) def refillcycles = tl.dataBits/(rowbits) def statebits = log2Up(states) + def nTSHRs = nReleaseTransactions + nAcquireTransactions require(states > 0) require(isPow2(sets)) @@ -60,6 +61,16 @@ abstract trait L2CacheBundle extends Bundle { override def clone = this.getClass.getConstructors.head.newInstance(l2cacheconf).asInstanceOf[this.type] } +trait HasL2Id extends L2CacheBundle { + val id = UInt(width = log2Up(l2cacheconf.nTSHRs)) +} + +trait HasL2InternalRequestState extends L2CacheBundle { + val tag_match = Bool() + val old_meta = new L2MetaData + val way_en = Bits(width = l2cacheconf.ways) +} + abstract class ReplacementPolicy { def way: UInt def miss: Unit @@ -80,20 +91,6 @@ abstract class MetaData(implicit val cacheconf: CacheConfig) extends CacheBundle val tag = Bits(width = cacheconf.tagbits) } -class L2MetaData(implicit val l2cacheconf: L2CacheConfig) extends MetaData - with L2CacheBundle { - val state = UInt(width = l2cacheconf.statebits) - val sharers = Bits(width = l2cacheconf.tl.ln.nClients) -} - -/* -class L3MetaData(implicit conf: L3CacheConfig) extends MetaData()(conf) { - val cstate = UInt(width = cacheconf.cstatebits) - val mstate = UInt(width = cacheconf.mstatebits) - val sharers = Bits(width = cacheconf.tl.ln.nClients) -} -*/ - class MetaReadReq(implicit val cacheconf: CacheConfig) extends CacheBundle { val idx = Bits(width = cacheconf.idxbits) } @@ -137,49 +134,125 @@ class MetaDataArray[T <: MetaData](gen: () => T)(implicit conf: CacheConfig) ext io.write.ready := !rst } -class L2DataReadReq(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle { +object L2MetaData { + def apply(tag: Bits, state: UInt, sharers: UInt)(implicit conf: L2CacheConfig) = { + val meta = new L2MetaData + meta.tag := tag + meta.state := state + meta.sharers := sharers + meta + } +} +class L2MetaData(implicit val l2cacheconf: L2CacheConfig) extends MetaData + with L2CacheBundle { + val state = UInt(width = l2cacheconf.statebits) + val sharers = Bits(width = l2cacheconf.tl.ln.nClients) +} + +/* +class L3MetaData(implicit conf: L3CacheConfig) extends MetaData()(conf) { + val cstate = UInt(width = cacheconf.cstatebits) + val mstate = UInt(width = cacheconf.mstatebits) + val sharers = Bits(width = cacheconf.tl.ln.nClients) +} +*/ + +class L2MetaReadReq(implicit val l2cacheconf: L2CacheConfig) extends MetaReadReq + with HasL2Id { + val tag = Bits(width = l2cacheconf.tagbits) +} + +class L2MetaWriteReq(implicit val l2cacheconf: L2CacheConfig) extends MetaWriteReq[L2MetaData](new L2MetaData) + with HasL2Id + +class L2MetaResp(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle + with HasL2Id + with HasL2InternalRequestState + +class L2MetaDataArray(implicit conf: L2CacheConfig) extends Module { + implicit val tl = conf.tl + val io = new Bundle { + val read = Decoupled(new L2MetaReadReq).flip + val write = Decoupled(new L2MetaWriteReq).flip + val resp = Valid(new L2MetaResp) + } + + val meta = Module(new MetaDataArray(() => L2MetaData(UInt(0), tl.co.newStateOnFlush, UInt(0)))) + meta.io.read <> io.read + meta.io.write <> io.write + + val s1_clk_en = Reg(next = io.read.fire()) + val s1_tag = RegEnable(io.read.bits.tag, io.read.valid) + val s1_id = RegEnable(io.read.bits.id, io.read.valid) + def wayMap[T <: Data](f: Int => T) = Vec((0 until conf.ways).map(f)) + val s1_tag_eq_way = wayMap((w: Int) => meta.io.resp(w).tag === s1_tag) + val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && tl.co.isValid(meta.io.resp(w).state)).toBits + val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) + val s2_tag_match = s2_tag_match_way.orR + val s2_hit_state = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).state, s1_clk_en))) + val s2_hit_sharers = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).sharers, s1_clk_en))) + + val replacer = new RandomReplacement + val s1_replaced_way_en = UIntToOH(replacer.way) + val s2_replaced_way_en = UIntToOH(RegEnable(replacer.way, s1_clk_en)) + val s2_repl_meta = Mux1H(s2_replaced_way_en, wayMap((w: Int) => + RegEnable(meta.io.resp(w), s1_clk_en && s1_replaced_way_en(w))).toSeq) + + io.resp.valid := Reg(next = s1_clk_en) + io.resp.bits.id := RegEnable(s1_id, s1_clk_en) + io.resp.bits.tag_match := s2_tag_match + io.resp.bits.old_meta := Mux(s2_tag_match, + L2MetaData(s2_repl_meta.tag, s2_hit_state, s2_hit_sharers), + s2_repl_meta) + io.resp.bits.way_en := Mux(s2_tag_match, s2_tag_match_way, s2_replaced_way_en) +} + +class L2DataReadReq(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle + with HasL2Id { val way_en = Bits(width = l2cacheconf.ways) val addr = Bits(width = l2cacheconf.tl.addrBits) } -class L2DataWriteReq(implicit conf: L2CacheConfig) extends L2DataReadReq()(conf) { - val wmask = Bits(width = conf.tl.writeMaskBits) - val data = Bits(width = conf.tl.dataBits) +class L2DataWriteReq(implicit l2cacheconf: L2CacheConfig) extends L2DataReadReq { + val wmask = Bits(width = l2cacheconf.tl.writeMaskBits) + val data = Bits(width = l2cacheconf.tl.dataBits) +} + +class L2DataResp(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle + with HasL2Id { + val data = Bits(width = l2cacheconf.tl.dataBits) } class L2DataArray(implicit conf: L2CacheConfig) extends Module { val io = new Bundle { val read = Decoupled(new L2DataReadReq).flip val write = Decoupled(new L2DataWriteReq).flip - val resp = Vec.fill(conf.ways){Bits(OUTPUT, conf.tl.dataBits)} + val resp = Valid(new L2DataResp) } val waddr = io.write.bits.addr val raddr = io.read.bits.addr val wmask = FillInterleaved(conf.wordbits, io.write.bits.wmask) - for (w <- 0 until conf.ways) { + val resp = (0 until conf.ways).map { w => val array = Mem(Bits(width=conf.rowbits), conf.sets*conf.refillcycles, seqRead = true) when (io.write.bits.way_en(w) && io.write.valid) { array.write(waddr, io.write.bits.data, wmask) } - io.resp(w) := array(RegEnable(raddr, io.read.bits.way_en(w) && io.read.valid)) + array(RegEnable(raddr, io.read.bits.way_en(w) && io.read.valid)) } + io.resp.valid := ShiftRegister(io.read.valid, 2) + io.resp.bits.id := ShiftRegister(io.read.bits.id, 2) + io.resp.bits.data := Mux1H(ShiftRegister(io.read.bits.way_en, 2), resp) io.read.ready := Bool(true) io.write.ready := Bool(true) } -trait L2InternalRequestState extends L2CacheBundle { - val tag_match = Bool() - val old_meta = new L2MetaData - val way_en = Bits(width = l2cacheconf.ways) -} - class L2InternalAcquire(implicit val l2cacheconf: L2CacheConfig) extends Acquire()(l2cacheconf.tl) - with L2InternalRequestState + with HasL2InternalRequestState class L2InternalRelease(implicit val l2cacheconf: L2CacheConfig) extends Release()(l2cacheconf.tl) - with L2InternalRequestState + with HasL2InternalRequestState class InternalTileLinkIO(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle { implicit val (tl, ln) = (l2cacheconf.tl, l2cacheconf.tl.ln) @@ -194,23 +267,16 @@ class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceA implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val tshrfile = Module(new TSHRFile(bankId)) - - // tags - val meta = Module(new MetaDataArray(() => new L2MetaData)) - - // data + val meta = Module(new L2MetaDataArray) val data = Module(new L2DataArray) - // replacement policy - val replacer = new RandomReplacement -/* - val s1_replaced_way_en = UIntToOH(replacer.way) - val s2_replaced_way_en = UIntToOH(RegEnable(replacer.way, s1_clk_en)) - val s2_repl_meta = Mux1H(s2_replaced_way_en, wayMap((w: Int) => - RegEnable(meta.io.resp(w), s1_clk_en && s1_replaced_way_en(w))).toSeq) -*/ - tshrfile.io.inner <> io.inner + tshrfile.io.meta_read <> meta.io.read + tshrfile.io.meta_write <> meta.io.write + tshrfile.io.meta_resp <> meta.io.resp + tshrfile.io.data_read <> data.io.read + tshrfile.io.data_write <> data.io.write + tshrfile.io.data_resp <> data.io.resp io.outer <> tshrfile.io.outer io.incoherent <> tshrfile.io.incoherent } @@ -219,19 +285,31 @@ class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceA class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val io = new Bundle { - val inner = (new InternalTileLinkIO).flip + val inner = (new TileLinkIO).flip val outer = new UncachedTileLinkIO val incoherent = Vec.fill(ln.nClients){Bool()}.asInput - val meta_read_req = Decoupled(new MetaReadReq) - val meta_write_req = Decoupled(new MetaWriteReq(new L2MetaData)) - val data_read_req = Decoupled(new L2DataReadReq) - val data_write_req = Decoupled(new L2DataWriteReq) + val meta_read = Decoupled(new L2MetaReadReq) + val meta_write = Decoupled(new L2MetaWriteReq) + val meta_resp = Valid(new L2MetaResp).flip + val data_read = Decoupled(new L2DataReadReq) + val data_write = Decoupled(new L2DataWriteReq) + val data_resp = Valid(new L2DataResp).flip + } + + // Wiring helper funcs + def doOutputArbitration[T <: Data](out: DecoupledIO[T], ins: Seq[DecoupledIO[T]]) { + val arb = Module(new RRArbiter(out.bits.clone, ins.size)) + out <> arb.io.out + arb.io.in zip ins map { case (a, i) => a <> i } } // Create TSHRs for outstanding transactions val nTrackers = conf.nReleaseTransactions + conf.nAcquireTransactions - val trackerList = (0 until conf.nReleaseTransactions).map(id => Module(new L2VoluntaryReleaseTracker(id, bankId))) ++ - (conf.nReleaseTransactions until nTrackers).map(id => Module(new L2AcquireTracker(id, bankId))) + val trackerList = (0 until conf.nReleaseTransactions).map { id => + Module(new L2VoluntaryReleaseTracker(id, bankId)) + } ++ (conf.nReleaseTransactions until nTrackers).map { id => + Module(new L2AcquireTracker(id, bankId)) + } // Propagate incoherence flags trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) @@ -251,10 +329,11 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && !block_acquires alloc_arb.io.out.ready := acquire.valid && !block_acquires - // Handle probe request generation - val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) - io.inner.probe <> probe_arb.io.out - probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.probe } + // Handle probe requests + doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) + //val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) + //io.inner.probe <> probe_arb.io.out + //probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.probe } // Handle releases, which might be voluntary and might have data val release = io.inner.release @@ -272,9 +351,10 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases // Reply to initial requestor - val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) - io.inner.grant <> grant_arb.io.out - grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } + doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) + //val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) + //io.inner.grant <> grant_arb.io.out + //grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } // Free finished transactions val ack = io.inner.finish @@ -282,21 +362,35 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { trackerList.map(_.io.inner.finish.bits := ack.bits) ack.ready := Bool(true) - // Create an arbiter for the one memory port + // Arbitrate for the outer memory port val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size)) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } io.outer <> outer_arb.io.out + + // Local memory + doOutputArbitration(io.meta_read, trackerList.map(_.io.meta_read)) + doOutputArbitration(io.meta_write, trackerList.map(_.io.meta_write)) + doOutputArbitration(io.data_read, trackerList.map(_.io.data_read)) + doOutputArbitration(io.data_write, trackerList.map(_.io.data_write)) + + } abstract class L2XactTracker()(implicit conf: L2CacheConfig) extends Module { implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val io = new Bundle { - val inner = (new InternalTileLinkIO).flip + val inner = (new TileLinkIO).flip val outer = new UncachedTileLinkIO val tile_incoherent = Bits(INPUT, ln.nClients) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) + val meta_read = Decoupled(new L2MetaReadReq) + val meta_write = Decoupled(new L2MetaWriteReq) + val meta_resp = Valid(new L2MetaResp).flip + val data_read = Decoupled(new L2DataReadReq) + val data_write = Decoupled(new L2DataWriteReq) + val data_resp = Valid(new L2DataResp).flip } val c_acq = io.inner.acquire.bits From 364a6de2145dd3e81cd432cf31fa56bb177eef3f Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sun, 18 May 2014 19:26:35 -0700 Subject: [PATCH 227/374] Use Mem instead of Vec[Reg] --- uncore/src/main/scala/htif.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 41d6df28..cc1c1c62 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -84,7 +84,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti val rx_word_count = (rx_count >> UInt(log2Up(short_request_bits/w))) val rx_word_done = io.host.in.valid && rx_count(log2Up(short_request_bits/w)-1,0).andR val packet_ram_depth = long_request_bits/short_request_bits-1 - val packet_ram = Vec.fill(packet_ram_depth){Reg(Bits(width = short_request_bits))} + val packet_ram = Mem(Bits(width = short_request_bits), packet_ram_depth) when (rx_word_done && io.host.in.ready) { packet_ram(rx_word_count(log2Up(packet_ram_depth)-1,0) - UInt(1)) := rx_shifter_in } From 3c329df7e761295b4d8e1b8b7bd3a41b32630271 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 28 May 2014 13:35:08 -0700 Subject: [PATCH 228/374] refactor Metadata, clean and expand coherence API --- uncore/src/main/scala/cache.scala | 102 ++-- uncore/src/main/scala/coherence.scala | 807 ++++++++++++++------------ uncore/src/main/scala/memserdes.scala | 2 +- uncore/src/main/scala/uncore.scala | 6 +- 4 files changed, 502 insertions(+), 415 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index e5851a6b..9a5afee3 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -67,7 +67,7 @@ trait HasL2Id extends L2CacheBundle { trait HasL2InternalRequestState extends L2CacheBundle { val tag_match = Bool() - val old_meta = new L2MetaData + val old_meta = new L2Metadata val way_en = Bits(width = l2cacheconf.ways) } @@ -87,46 +87,45 @@ class RandomReplacement(implicit val cacheconf: CacheConfig) extends Replacement def hit = {} } -abstract class MetaData(implicit val cacheconf: CacheConfig) extends CacheBundle { +abstract class Metadata(implicit val cacheconf: CacheConfig) extends CacheBundle { val tag = Bits(width = cacheconf.tagbits) + val coh: CoherenceMetadata } class MetaReadReq(implicit val cacheconf: CacheConfig) extends CacheBundle { val idx = Bits(width = cacheconf.idxbits) } -class MetaWriteReq[T <: MetaData](gen: T)(implicit conf: CacheConfig) extends MetaReadReq { +class MetaWriteReq[T <: Metadata](gen: T)(implicit conf: CacheConfig) extends MetaReadReq { val way_en = Bits(width = conf.ways) val data = gen.clone override def clone = new MetaWriteReq(gen)(conf).asInstanceOf[this.type] } -class MetaDataArray[T <: MetaData](gen: () => T)(implicit conf: CacheConfig) extends Module { +class MetadataArray[T <: Metadata](makeRstVal: () => T)(implicit conf: CacheConfig) extends Module { implicit val tl = conf.tl - val rstVal = gen() + val rstVal = makeRstVal() val io = new Bundle { val read = Decoupled(new MetaReadReq).flip val write = Decoupled(new MetaWriteReq(rstVal.clone)).flip val resp = Vec.fill(conf.ways){rstVal.clone.asOutput} } - val metabits = rstVal.getWidth val rst_cnt = Reg(init=UInt(0, log2Up(conf.sets+1))) val rst = rst_cnt < UInt(conf.sets) + val waddr = Mux(rst, rst_cnt, io.write.bits.idx) + val wdata = Mux(rst, rstVal, io.write.bits.data).toBits + val wmask = Mux(rst, SInt(-1), io.write.bits.way_en) when (rst) { rst_cnt := rst_cnt+UInt(1) } - val tags = Mem(UInt(width = metabits*conf.ways), conf.sets, seqRead = true) - + val tag_arr = Mem(UInt(width = metabits*conf.ways), conf.sets, seqRead = true) when (rst || io.write.valid) { - val addr = Mux(rst, rst_cnt, io.write.bits.idx) - val data = Mux(rst, rstVal, io.write.bits.data).toBits - val mask = Mux(rst, SInt(-1), io.write.bits.way_en) - tags.write(addr, Fill(conf.ways, data), FillInterleaved(metabits, mask)) + tag_arr.write(waddr, Fill(conf.ways, wdata), FillInterleaved(metabits, wmask)) } - val tag = tags(RegEnable(io.read.bits.idx, io.read.valid)) + val tags = tag_arr(RegEnable(io.read.bits.idx, io.read.valid)) for (w <- 0 until conf.ways) { - val m = tag(metabits*(w+1)-1, metabits*w) + val m = tags(metabits*(w+1)-1, metabits*w) io.resp(w) := rstVal.clone.fromBits(m) } @@ -134,26 +133,23 @@ class MetaDataArray[T <: MetaData](gen: () => T)(implicit conf: CacheConfig) ext io.write.ready := !rst } -object L2MetaData { - def apply(tag: Bits, state: UInt, sharers: UInt)(implicit conf: L2CacheConfig) = { - val meta = new L2MetaData +object L2Metadata { + def apply(tag: Bits, coh: MasterMetadata)(implicit conf: L2CacheConfig) = { + val meta = new L2Metadata meta.tag := tag - meta.state := state - meta.sharers := sharers + meta.coh := coh meta } } -class L2MetaData(implicit val l2cacheconf: L2CacheConfig) extends MetaData +class L2Metadata(implicit val l2cacheconf: L2CacheConfig) extends Metadata with L2CacheBundle { - val state = UInt(width = l2cacheconf.statebits) - val sharers = Bits(width = l2cacheconf.tl.ln.nClients) + val coh = l2cacheconf.tl.co.masterMetadataOnFlush.clone } /* -class L3MetaData(implicit conf: L3CacheConfig) extends MetaData()(conf) { - val cstate = UInt(width = cacheconf.cstatebits) - val mstate = UInt(width = cacheconf.mstatebits) - val sharers = Bits(width = cacheconf.tl.ln.nClients) +class L3Metadata(implicit conf: L3CacheConfig) extends Metadata + with L3CacheBundle { + val coh = MixedMetaData(conf.innerTL.co, conf.outerTL.co) ? } */ @@ -162,14 +158,14 @@ class L2MetaReadReq(implicit val l2cacheconf: L2CacheConfig) extends MetaReadReq val tag = Bits(width = l2cacheconf.tagbits) } -class L2MetaWriteReq(implicit val l2cacheconf: L2CacheConfig) extends MetaWriteReq[L2MetaData](new L2MetaData) +class L2MetaWriteReq(implicit val l2cacheconf: L2CacheConfig) extends MetaWriteReq[L2Metadata](new L2Metadata) with HasL2Id class L2MetaResp(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle with HasL2Id with HasL2InternalRequestState -class L2MetaDataArray(implicit conf: L2CacheConfig) extends Module { +class L2MetadataArray(implicit conf: L2CacheConfig) extends Module { implicit val tl = conf.tl val io = new Bundle { val read = Decoupled(new L2MetaReadReq).flip @@ -177,7 +173,7 @@ class L2MetaDataArray(implicit conf: L2CacheConfig) extends Module { val resp = Valid(new L2MetaResp) } - val meta = Module(new MetaDataArray(() => L2MetaData(UInt(0), tl.co.newStateOnFlush, UInt(0)))) + val meta = Module(new MetadataArray(() => L2Metadata(UInt(0), tl.co.masterMetadataOnFlush))) meta.io.read <> io.read meta.io.write <> io.write @@ -186,11 +182,11 @@ class L2MetaDataArray(implicit conf: L2CacheConfig) extends Module { val s1_id = RegEnable(io.read.bits.id, io.read.valid) def wayMap[T <: Data](f: Int => T) = Vec((0 until conf.ways).map(f)) val s1_tag_eq_way = wayMap((w: Int) => meta.io.resp(w).tag === s1_tag) - val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && tl.co.isValid(meta.io.resp(w).state)).toBits + val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && tl.co.isValid(meta.io.resp(w).coh)).toBits val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) val s2_tag_match = s2_tag_match_way.orR - val s2_hit_state = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).state, s1_clk_en))) - val s2_hit_sharers = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).sharers, s1_clk_en))) + val s2_hit_coh = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).coh, s1_clk_en))) + //val s2_hit = s2_tag_match && tl.co.isHit(s2_req.cmd, s2_hit_state) && s2_hit_state === tl.co.newStateOnHit(s2_req.cmd, s2_hit_state) val replacer = new RandomReplacement val s1_replaced_way_en = UIntToOH(replacer.way) @@ -202,7 +198,7 @@ class L2MetaDataArray(implicit conf: L2CacheConfig) extends Module { io.resp.bits.id := RegEnable(s1_id, s1_clk_en) io.resp.bits.tag_match := s2_tag_match io.resp.bits.old_meta := Mux(s2_tag_match, - L2MetaData(s2_repl_meta.tag, s2_hit_state, s2_hit_sharers), + L2Metadata(s2_repl_meta.tag, s2_hit_coh), s2_repl_meta) io.resp.bits.way_en := Mux(s2_tag_match, s2_tag_match_way, s2_replaced_way_en) } @@ -267,7 +263,7 @@ class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceA implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val tshrfile = Module(new TSHRFile(bankId)) - val meta = Module(new L2MetaDataArray) + val meta = Module(new L2MetadataArray) val data = Module(new L2DataArray) tshrfile.io.inner <> io.inner @@ -300,7 +296,12 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { def doOutputArbitration[T <: Data](out: DecoupledIO[T], ins: Seq[DecoupledIO[T]]) { val arb = Module(new RRArbiter(out.bits.clone, ins.size)) out <> arb.io.out - arb.io.in zip ins map { case (a, i) => a <> i } + arb.io.in zip ins map { case (a, in) => a <> in } + } + + def doInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { + outs.map(_.bits := in.bits) + outs.zipWithIndex.map { case (o, i) => o.valid := UInt(i) === in.bits.id } } // Create TSHRs for outstanding transactions @@ -331,9 +332,6 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { // Handle probe requests doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) - //val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) - //io.inner.probe <> probe_arb.io.out - //probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.probe } // Handle releases, which might be voluntary and might have data val release = io.inner.release @@ -352,9 +350,6 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { // Reply to initial requestor doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) - //val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) - //io.inner.grant <> grant_arb.io.out - //grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } // Free finished transactions val ack = io.inner.finish @@ -372,7 +367,8 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { doOutputArbitration(io.meta_write, trackerList.map(_.io.meta_write)) doOutputArbitration(io.data_read, trackerList.map(_.io.data_read)) doOutputArbitration(io.data_write, trackerList.map(_.io.data_write)) - + doInputRouting(io.meta_resp, trackerList.map(_.io.meta_resp)) + doInputRouting(io.data_resp, trackerList.map(_.io.data_resp)) } @@ -406,6 +402,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Ca val state = Reg(init=s_idle) val xact = Reg{ new Release } val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val new_meta = Reg(new L2Metadata) val incoming_rel = io.inner.release.bits io.has_acquire_conflict := Bool(false) @@ -430,15 +427,28 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Ca xact.client_xact_id, UInt(trackerId)) + switch (state) { is(s_idle) { io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { xact := incoming_rel.payload init_client_id := incoming_rel.header.src - state := Mux(co.messageHasData(incoming_rel.payload), s_mem, s_ack) + state := s_mem } } +/* + is(s_meta_read) { + when(io.meta_read.ready) state := s_meta_resp + } + is(s_meta_resp) { + when(io.meta_resp.valid) { + new_meta := L2Metadata(io.meta.resp.bits.old_meta.tag, io.meta.resp.bits.old_meta.sharers, io.meta.resp.bits + old_meta := io.meta.resp.bits.old_meta + state := Mux(s_meta_write +Mux(co.messageHasData(xact), s_mem, s_ack) + } + */ is(s_mem) { io.outer.acquire.valid := Bool(true) when(io.outer.acquire.ready) { state := s_ack } @@ -463,7 +473,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig val curr_p_id = PriorityEncoder(probe_flags) val pending_outer_write = co.messageHasData(xact) - val pending_outer_read = co.needsOuterRead(xact.a_type, UInt(0)) + val pending_outer_read = co.requiresOuterRead(xact.a_type) val outer_write_acq = Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId), xact.data) val outer_write_rel = Acquire(co.getUncachedWriteAcquireType, @@ -491,7 +501,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact.a_type, UInt(0)), + io.inner.probe.bits.payload := Probe(co.getProbeType(xact.a_type, conf.tl.co.masterMetadataOnFlush), xact.addr, UInt(trackerId)) @@ -511,7 +521,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig is(s_idle) { io.inner.acquire.ready := Bool(true) val needs_outer_write = co.messageHasData(c_acq.payload) - val needs_outer_read = co.needsOuterRead(c_acq.payload.a_type, UInt(0)) + val needs_outer_read = co.requiresOuterRead(c_acq.payload.a_type) when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 327e31e5..45a1970c 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -1,7 +1,86 @@ package uncore import Chisel._ -abstract class CoherencePolicy { +object MuxBundle { + def apply[T <: Data] (default: T, mapping: Seq[(Bool, T)]): T = { + var res = default; + for ((t, v) <- mapping.reverse){ + res = Mux(t, v, res); + } + res + } +} + +abstract class CoherenceMetadata extends Bundle + +object ClientMetadata { + def apply(state: UInt)(implicit c: CoherencePolicy) = { + val m = new ClientMetadata + m.state := state + m + } +} +class ClientMetadata(implicit c: CoherencePolicy) extends CoherenceMetadata { + val state = UInt(width = c.clientStateWidth) + def ===(right: ClientMetadata): Bool = this.state === right.state + override def clone = new ClientMetadata()(c).asInstanceOf[this.type] +} + +object MasterMetadata { + def apply(state: UInt)(implicit c: CoherencePolicy): MasterMetadata = { + val m = new MasterMetadata + m.state := state + m.sharers := m.sharers.flush() + m + } + def apply(state: UInt, sharers: DirectoryRepresentation)(implicit c: CoherencePolicy): MasterMetadata = { + val m = apply(state) + m.sharers := sharers + m + } +} +class MasterMetadata(implicit c: CoherencePolicy) extends CoherenceMetadata { + val state = UInt(width = c.masterStateWidth) + val sharers = c.dir.clone + override def clone = new MasterMetadata()(c).asInstanceOf[this.type] +} +/* +class MixedMetadata(inner: CoherencePolicy, outer: CoherencePolicy) extends CoherenceMetadata { + val cstate = UInt(width = outer.clientStateWidth) + val mstate = UInt(width = inner.masterStateWidth) + val sharers = inner.dir.sharers.clone +} +*/ + +abstract class DirectoryRepresentation extends Bundle { + val sharers: UInt + def pop(id: UInt): DirectoryRepresentation + def push(id: UInt): DirectoryRepresentation + def flush(dummy: Int = 0): DirectoryRepresentation + def none(dummy: Int = 0): Bool + def one(dummy: Int = 0): Bool +} + +class NullRepresentation extends DirectoryRepresentation { + val sharers = UInt(0) + def pop(id: UInt) = this + def push(id: UInt) = this + def flush(dummy: Int = 0) = this + def none(dummy: Int = 0) = Bool(false) + def one(dummy: Int = 0) = Bool(false) +} + +class FullRepresentation(nClients: Int) extends DirectoryRepresentation { + val sharers = UInt(width = nClients) + def pop(id: UInt) = { sharers := sharers & ~UIntToOH(id); this } + def push(id: UInt) = { sharers := sharers | UIntToOH(id); this } + def flush(dummy: Int = 0) = { sharers := UInt(0, width = nClients); this } + def none(dummy: Int = 0) = sharers === UInt(0) + def one(dummy: Int = 0) = PopCount(sharers) === UInt(1) + override def clone = new FullRepresentation(nClients).asInstanceOf[this.type] +} + +abstract class CoherencePolicy(val dir: DirectoryRepresentation) { def nClientStates: Int def nMasterStates: Int def nAcquireTypes: Int @@ -15,26 +94,29 @@ abstract class CoherencePolicy { def releaseTypeWidth = log2Up(nReleaseTypes) def grantTypeWidth = log2Up(nGrantTypes) - def isHit (cmd: UInt, state: UInt): Bool - def isValid (state: UInt): Bool + def isHit (cmd: UInt, m: ClientMetadata): Bool + def isValid (m: ClientMetadata): Bool + def isHit (incoming: Acquire, m: MasterMetadata): Bool + def isValid (m: MasterMetadata): Bool def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool - def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool - def needsWriteback (state: UInt): Bool + def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool + def needsWriteback(m: ClientMetadata): Bool - def newStateOnHit(cmd: UInt, state: UInt): UInt - def newStateOnCacheControl(cmd: UInt): UInt - def newStateOnWriteback(): UInt - def newStateOnFlush(): UInt - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt - def newStateOnProbe(incoming: Probe, state: UInt): UInt + def clientMetadataOnHit(cmd: UInt, m: ClientMetadata): ClientMetadata + def clientMetadataOnCacheControl(cmd: UInt): ClientMetadata + def clientMetadataOnFlush: ClientMetadata + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire): ClientMetadata + def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata): ClientMetadata + def masterMetadataOnFlush: MasterMetadata + def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt): MasterMetadata - def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt - def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt - def getProbeType(a_type: UInt, global_state: UInt): UInt + def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt + def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt + def getProbeType(a_type: UInt, m: MasterMetadata): UInt def getReleaseTypeOnCacheControl(cmd: UInt): UInt def getReleaseTypeOnVoluntaryWriteback(): UInt - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt + def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt def getGrantType(a_type: UInt, count: UInt): UInt def getGrantType(rel: Release, count: UInt): UInt @@ -45,9 +127,10 @@ abstract class CoherencePolicy { def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool def isVoluntary(rel: Release): Bool def isVoluntary(gnt: Grant): Bool - def needsOuterRead(a_type: UInt, global_state: UInt): Bool - def needsOuterWrite(a_type: UInt, global_state: UInt): Bool - def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool + def requiresOuterRead(a_type: UInt, m: MasterMetadata): Bool + def requiresOuterWrite(a_type: UInt, m: MasterMetadata): Bool + def requiresOuterRead(a_type: UInt): Bool + def requiresOuterWrite(a_type: UInt): Bool def requiresSelfProbe(a_type: UInt): Bool def requiresAckForGrant(g_type: UInt): Bool def requiresAckForRelease(r_type: UInt): Bool @@ -63,85 +146,10 @@ trait UncachedTransactions { def isUncachedReadTransaction(acq: Acquire): Bool } -abstract class CoherencePolicyWithUncached extends CoherencePolicy with UncachedTransactions +abstract class CoherencePolicyWithUncached(dir: DirectoryRepresentation) extends CoherencePolicy(dir) + with UncachedTransactions -abstract class IncoherentPolicy extends CoherencePolicy { - // UNIMPLEMENTED - def newStateOnProbe(incoming: Probe, state: UInt): UInt = state - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = UInt(0) - def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = Bool(false) - def getGrantType(a_type: UInt, count: UInt): UInt = UInt(0) - def getGrantType(rel: Release, count: UInt): UInt = UInt(0) - def getProbeType(a_type: UInt, global_state: UInt): UInt = UInt(0) - def needsOuterRead(a_type: UInt, global_state: UInt): Bool = Bool(false) - def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = Bool(false) - def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) - def requiresAckForGrant(g_type: UInt) = Bool(true) - def requiresAckForRelease(r_type: UInt) = Bool(false) - def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = Bool(false) - -} - -class ThreeStateIncoherence extends IncoherentPolicy { - def nClientStates = 3 - def nMasterStates = 0 - def nAcquireTypes = 3 - def nProbeTypes = 0 - def nReleaseTypes = 2 - def nGrantTypes = 3 - val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(UInt(), nClientStates) - val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(UInt(), nAcquireTypes) - val releaseVoluntaryInvalidateData :: releaseInvalidateAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantData :: grantAck :: Nil = Enum(UInt(), nGrantTypes) - val hasDataAcquireTypeVec = Vec(acquireWriteback) - val hasDataReleaseTypeVec = Vec(acquireWriteback) - val hasDataGrantTypeVec = Vec(grantData) - - def isHit ( cmd: UInt, state: UInt): Bool = (state === tileClean || state === tileDirty) - def isValid (state: UInt): Bool = state != tileInvalid - - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire) = Bool(false) - def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = state === tileDirty - def needsWriteback (state: UInt): Bool = state === tileDirty - - def newState(cmd: UInt, state: UInt): UInt = { - Mux(isWrite(cmd), tileDirty, Mux(isRead(cmd), Mux(state === tileDirty, tileDirty, tileClean), state)) - } - def newStateOnHit(cmd: UInt, state: UInt): UInt = newState(cmd, state) - def newStateOnCacheControl(cmd: UInt) = tileInvalid //TODO - def newStateOnWriteback() = tileInvalid - def newStateOnFlush() = tileInvalid - def newStateOnGrant(incoming: Grant, outstanding: Acquire) = { - MuxLookup(incoming.g_type, tileInvalid, Array( - grantData -> Mux(outstanding.a_type === acquireReadDirty, tileDirty, tileClean), - grantAck -> tileInvalid - )) - } - - def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = { - Mux(isWriteIntent(cmd), acquireReadDirty, acquireReadClean) - } - def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { - Mux(isWriteIntent(cmd), acquireReadDirty, outstanding.a_type) - } - def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): UInt = releaseVoluntaryInvalidateData - - def messageHasData( msg: SourcedMessage ) = msg match { - case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) - case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) - case rel: Release => Bool(false) - case _ => Bool(false) - } - def messageUpdatesDataArray (reply: Grant) = (reply.g_type === grantData) - def messageIsUncached(acq: Acquire): Bool = Bool(false) -} - -class MICoherence extends CoherencePolicyWithUncached { +class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 2 def nMasterStates = 2 def nAcquireTypes = 6 @@ -149,8 +157,8 @@ class MICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 5 def nGrantTypes = 7 - val tileInvalid :: tileValid :: Nil = Enum(UInt(), nClientStates) - val globalInvalid :: globalValid :: Nil = Enum(UInt(), nMasterStates) + val clientInvalid :: clientValid :: Nil = Enum(UInt(), nClientStates) + val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) @@ -162,43 +170,54 @@ class MICoherence extends CoherencePolicyWithUncached { val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseCopyData) val hasDataGrantTypeVec = Vec(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) - def isHit (cmd: UInt, state: UInt): Bool = state != tileInvalid - def isValid (state: UInt): Bool = state != tileInvalid + def isHit (cmd: UInt, m: ClientMetadata): Bool = isValid(m) + def isValid (m: ClientMetadata): Bool = m.state != clientInvalid + def isHit (incoming: Acquire, m: MasterMetadata): Bool = isValid(m) + def isValid (m: MasterMetadata): Bool = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = (outstanding.a_type != acquireReadExclusive) - def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { - MuxLookup(cmd, (state === tileValid), Array( - M_INV -> (state === tileValid), - M_CLN -> (state === tileValid) + def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { + MuxLookup(cmd, (m.state === clientValid), Array( + M_INV -> (m.state === clientValid), + M_CLN -> (m.state === clientValid) )) } - def needsWriteback (state: UInt): Bool = { - needsTransactionOnCacheControl(M_INV, state) + def needsWriteback (m: ClientMetadata): Bool = { + needsTransactionOnCacheControl(M_INV, m) } - def newStateOnHit(cmd: UInt, state: UInt): UInt = state - def newStateOnCacheControl(cmd: UInt) = { - MuxLookup(cmd, tileInvalid, Array( - M_INV -> tileInvalid, - M_CLN -> tileValid - )) - } - def newStateOnWriteback() = newStateOnCacheControl(M_INV) - def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt = { - MuxLookup(incoming.g_type, tileInvalid, Array( - grantReadExclusive -> tileValid, - grantReadUncached -> tileInvalid, - grantWriteUncached -> tileInvalid, - grantReadWordUncached -> tileInvalid, - grantWriteWordUncached -> tileInvalid, - grantAtomicUncached -> tileInvalid - )) - } - def newStateOnProbe(incoming: Probe, state: UInt): UInt = { - MuxLookup(incoming.p_type, state, Array( - probeInvalidate -> tileInvalid, - probeCopy -> state + def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = m + def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( + MuxLookup(cmd, clientInvalid, Array( + M_INV -> clientInvalid, + M_CLN -> clientValid + )))(this) + def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( + MuxLookup(incoming.g_type, clientInvalid, Array( + grantReadExclusive -> clientValid, + grantReadUncached -> clientInvalid, + grantWriteUncached -> clientInvalid, + grantReadWordUncached -> clientInvalid, + grantWriteWordUncached -> clientInvalid, + grantAtomicUncached -> clientInvalid + )))(this) + def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( + MuxLookup(incoming.p_type, m.state, Array( + probeInvalidate -> clientInvalid, + probeCopy -> m.state + )))(this) + def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) + def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt) = { + val popped = m.sharers.pop(src) + val next = MasterMetadata(Mux(popped.none(), masterInvalid, masterValid), popped)(this) + def is(r: UInt) = incoming.r_type === r + MuxBundle(m, Array( + is(releaseVoluntaryInvalidateData) -> next, + is(releaseInvalidateData) -> next, + is(releaseCopyData) -> m, + is(releaseInvalidateAck) -> next, + is(releaseCopyAck) -> m )) } @@ -211,11 +230,11 @@ class MICoherence extends CoherencePolicyWithUncached { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = acquireReadExclusive - def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = acquireReadExclusive + def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = acquireReadExclusive + def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = acquireReadExclusive def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { + def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeCopy -> releaseCopyData @@ -224,7 +243,7 @@ class MICoherence extends CoherencePolicyWithUncached { probeInvalidate -> releaseInvalidateAck, probeCopy -> releaseCopyAck )) - Mux(needsWriteback(state), with_data, without_data) + Mux(needsWriteback(m), with_data, without_data) } def messageHasData(msg: SourcedMessage) = msg match { @@ -257,7 +276,7 @@ class MICoherence extends CoherencePolicyWithUncached { )) } - def getProbeType(a_type: UInt, global_state: UInt): UInt = { + def getProbeType(a_type: UInt, m: MasterMetadata): UInt = { MuxLookup(a_type, probeCopy, Array( acquireReadExclusive -> probeInvalidate, acquireReadUncached -> probeCopy, @@ -268,22 +287,21 @@ class MICoherence extends CoherencePolicyWithUncached { )) } - def needsOuterRead(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterRead(a_type: UInt) = { (a_type != acquireWriteUncached) } - def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterWrite(a_type: UInt) = { (a_type === acquireWriteUncached) } - def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { - needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) - } + def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) + def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MEICoherence extends CoherencePolicyWithUncached { +class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 3 def nMasterStates = 2 def nAcquireTypes = 7 @@ -291,8 +309,8 @@ class MEICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 7 def nGrantTypes = 8 - val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - val globalInvalid :: globalExclusiveClean :: Nil = Enum(UInt(), nMasterStates) + val clientInvalid :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) + val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) @@ -304,50 +322,64 @@ class MEICoherence extends CoherencePolicyWithUncached { val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) val hasDataGrantTypeVec = Vec(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) - def isHit (cmd: UInt, state: UInt): Bool = state != tileInvalid - def isValid (state: UInt): Bool = state != tileInvalid + def isHit (cmd: UInt, m: ClientMetadata) = isValid(m) + def isValid (m: ClientMetadata) = m.state != clientInvalid + def isHit (incoming: Acquire, m: MasterMetadata) = isValid(m) + def isValid (m: MasterMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && messageIsUncached(outstanding)) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusiveDirty)) } - def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { - MuxLookup(cmd, (state === tileExclusiveDirty), Array( - M_INV -> (state === tileExclusiveDirty), - M_CLN -> (state === tileExclusiveDirty) + def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { + MuxLookup(cmd, (m.state === clientExclusiveDirty), Array( + M_INV -> (m.state === clientExclusiveDirty), + M_CLN -> (m.state === clientExclusiveDirty) )) } - def needsWriteback (state: UInt): Bool = { - needsTransactionOnCacheControl(M_INV, state) + def needsWriteback (m: ClientMetadata): Bool = { + needsTransactionOnCacheControl(M_INV, m) } - def newStateOnHit(cmd: UInt, state: UInt): UInt = { - Mux(isWrite(cmd), tileExclusiveDirty, state) - } - def newStateOnCacheControl(cmd: UInt) = { - MuxLookup(cmd, tileInvalid, Array( - M_INV -> tileInvalid, - M_CLN -> tileExclusiveClean - )) - } - def newStateOnWriteback() = newStateOnCacheControl(M_INV) - def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt = { - MuxLookup(incoming.g_type, tileInvalid, Array( - grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusiveDirty, tileExclusiveDirty, tileExclusiveClean), - grantReadExclusiveAck -> tileExclusiveDirty, - grantReadUncached -> tileInvalid, - grantWriteUncached -> tileInvalid, - grantReadWordUncached -> tileInvalid, - grantWriteWordUncached -> tileInvalid, - grantAtomicUncached -> tileInvalid - )) - } - def newStateOnProbe(incoming: Probe, state: UInt): UInt = { - MuxLookup(incoming.p_type, state, Array( - probeInvalidate -> tileInvalid, - probeDowngrade -> tileExclusiveClean, - probeCopy -> state + def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state))(this) + + def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( + MuxLookup(cmd, clientInvalid, Array( + M_INV -> clientInvalid, + M_CLN -> clientExclusiveClean + )))(this) + def clientMetadataOnFlush() = clientMetadataOnCacheControl(M_INV) + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( + MuxLookup(incoming.g_type, clientInvalid, Array( + grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusiveDirty, + clientExclusiveDirty, clientExclusiveClean), + grantReadExclusiveAck -> clientExclusiveDirty, + grantReadUncached -> clientInvalid, + grantWriteUncached -> clientInvalid, + grantReadWordUncached -> clientInvalid, + grantWriteWordUncached -> clientInvalid, + grantAtomicUncached -> clientInvalid + )))(this) + def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( + MuxLookup(incoming.p_type, m.state, Array( + probeInvalidate -> clientInvalid, + probeDowngrade -> clientExclusiveClean, + probeCopy -> m.state + )))(this) + def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) + def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt) = { + val popped = m.sharers.pop(src) + val next = MasterMetadata(Mux(popped.none(), masterInvalid, masterValid), popped)(this) + def is(r: UInt) = incoming.r_type === r + MuxBundle(m, Array( + is(releaseVoluntaryInvalidateData) -> next, + is(releaseInvalidateData) -> next, + is(releaseDowngradeData) -> m, + is(releaseCopyData) -> m, + is(releaseInvalidateAck) -> next, + is(releaseDowngradeAck) -> m, + is(releaseCopyAck) -> m )) } @@ -360,15 +392,15 @@ class MEICoherence extends CoherencePolicyWithUncached { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = { + def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, acquireReadExclusiveClean) } - def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { + def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, outstanding.a_type) } def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { + def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -379,7 +411,7 @@ class MEICoherence extends CoherencePolicyWithUncached { probeDowngrade -> releaseDowngradeAck, probeCopy -> releaseCopyAck )) - Mux(needsWriteback(state), with_data, without_data) + Mux(needsWriteback(m), with_data, without_data) } def messageHasData(msg: SourcedMessage) = msg match { @@ -413,7 +445,7 @@ class MEICoherence extends CoherencePolicyWithUncached { } - def getProbeType(a_type: UInt, global_state: UInt): UInt = { + def getProbeType(a_type: UInt, m: MasterMetadata): UInt = { MuxLookup(a_type, probeCopy, Array( acquireReadExclusiveClean -> probeInvalidate, acquireReadExclusiveDirty -> probeInvalidate, @@ -425,15 +457,14 @@ class MEICoherence extends CoherencePolicyWithUncached { )) } - def needsOuterRead(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterRead(a_type: UInt) = { (a_type != acquireWriteUncached) } - def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterWrite(a_type: UInt) = { (a_type === acquireWriteUncached) } - def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { - needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) - } + def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) + def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = Bool(false) @@ -441,7 +472,7 @@ class MEICoherence extends CoherencePolicyWithUncached { def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MSICoherence extends CoherencePolicyWithUncached { +class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 3 def nMasterStates = 3 def nAcquireTypes = 7 @@ -449,8 +480,8 @@ class MSICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 7 def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(UInt(), nMasterStates) + val clientInvalid :: clientShared :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) + val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) @@ -462,56 +493,70 @@ class MSICoherence extends CoherencePolicyWithUncached { val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) - def isHit (cmd: UInt, state: UInt): Bool = { - Mux(isWriteIntent(cmd), (state === tileExclusiveDirty), - (state === tileShared || state === tileExclusiveDirty)) + def isHit (cmd: UInt, m: ClientMetadata): Bool = { + Mux(isWriteIntent(cmd), (m.state === clientExclusiveDirty), + (m.state === clientShared || m.state === clientExclusiveDirty)) } - def isValid (state: UInt): Bool = { - state != tileInvalid + def isValid (m: ClientMetadata): Bool = { + m.state != clientInvalid } + def isHit (incoming: Acquire, m: MasterMetadata) = isValid(m) + def isValid (m: MasterMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && messageIsUncached(outstanding)) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) } - def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { - MuxLookup(cmd, (state === tileExclusiveDirty), Array( - M_INV -> (state === tileExclusiveDirty), - M_CLN -> (state === tileExclusiveDirty) + def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { + MuxLookup(cmd, (m.state === clientExclusiveDirty), Array( + M_INV -> (m.state === clientExclusiveDirty), + M_CLN -> (m.state === clientExclusiveDirty) )) } - def needsWriteback (state: UInt): Bool = { - needsTransactionOnCacheControl(M_INV, state) + def needsWriteback (m: ClientMetadata): Bool = { + needsTransactionOnCacheControl(M_INV, m) } - def newStateOnHit(cmd: UInt, state: UInt): UInt = { - Mux(isWrite(cmd), tileExclusiveDirty, state) - } - def newStateOnCacheControl(cmd: UInt) = { - MuxLookup(cmd, tileInvalid, Array( - M_INV -> tileInvalid, - M_CLN -> tileShared - )) - } - def newStateOnWriteback() = newStateOnCacheControl(M_INV) - def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt = { - MuxLookup(incoming.g_type, tileInvalid, Array( - grantReadShared -> tileShared, - grantReadExclusive -> tileExclusiveDirty, - grantReadExclusiveAck -> tileExclusiveDirty, - grantReadUncached -> tileInvalid, - grantWriteUncached -> tileInvalid, - grantReadWordUncached -> tileInvalid, - grantWriteWordUncached -> tileInvalid, - grantAtomicUncached -> tileInvalid - )) - } - def newStateOnProbe(incoming: Probe, state: UInt): UInt = { - MuxLookup(incoming.p_type, state, Array( - probeInvalidate -> tileInvalid, - probeDowngrade -> tileShared, - probeCopy -> state + def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state))(this) + def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( + MuxLookup(cmd, clientInvalid, Array( + M_INV -> clientInvalid, + M_CLN -> clientShared + )))(this) + def clientMetadataOnFlush() = clientMetadataOnCacheControl(M_INV) + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( + MuxLookup(incoming.g_type, clientInvalid, Array( + grantReadShared -> clientShared, + grantReadExclusive -> clientExclusiveDirty, + grantReadExclusiveAck -> clientExclusiveDirty, + grantReadUncached -> clientInvalid, + grantWriteUncached -> clientInvalid, + grantReadWordUncached -> clientInvalid, + grantWriteWordUncached -> clientInvalid, + grantAtomicUncached -> clientInvalid + )))(this) + def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( + MuxLookup(incoming.p_type, m.state, Array( + probeInvalidate -> clientInvalid, + probeDowngrade -> clientShared, + probeCopy -> m.state + )))(this) + def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) + def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt) = { + val popped = m.sharers.pop(src) + val next = MasterMetadata( + Mux(popped.none(), masterInvalid, + Mux(popped.one(), masterExclusive, masterShared)), popped)(this) + def is(r: UInt) = incoming.r_type === r + MuxBundle(m, Array( + is(releaseVoluntaryInvalidateData) -> next, + is(releaseInvalidateData) -> next, + is(releaseDowngradeData) -> m, + is(releaseCopyData) -> m, + is(releaseInvalidateAck) -> next, + is(releaseDowngradeAck) -> m, + is(releaseCopyAck) -> m )) } @@ -524,15 +569,15 @@ class MSICoherence extends CoherencePolicyWithUncached { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = { + def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) } - def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { + def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, outstanding.a_type) } def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { + def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -543,7 +588,7 @@ class MSICoherence extends CoherencePolicyWithUncached { probeDowngrade -> releaseDowngradeAck, probeCopy -> releaseCopyAck )) - Mux(needsWriteback(state), with_data, without_data) + Mux(needsWriteback(m), with_data, without_data) } def messageHasData(msg: SourcedMessage) = msg match { @@ -576,8 +621,7 @@ class MSICoherence extends CoherencePolicyWithUncached { )) } - - def getProbeType(a_type: UInt, global_state: UInt): UInt = { + def getProbeType(a_type: UInt, m: MasterMetadata): UInt = { MuxLookup(a_type, probeCopy, Array( acquireReadShared -> probeDowngrade, acquireReadExclusive -> probeInvalidate, @@ -586,15 +630,14 @@ class MSICoherence extends CoherencePolicyWithUncached { )) } - def needsOuterRead(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterRead(a_type: UInt) = { (a_type != acquireWriteUncached) } - def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterWrite(a_type: UInt) = { (a_type === acquireWriteUncached) } - def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { - needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) - } + def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) + def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = Bool(false) @@ -602,7 +645,7 @@ class MSICoherence extends CoherencePolicyWithUncached { def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MESICoherence extends CoherencePolicyWithUncached { +class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 4 def nMasterStates = 3 def nAcquireTypes = 7 @@ -610,8 +653,8 @@ class MESICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 7 def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(UInt(), nMasterStates) + val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) + val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) @@ -623,56 +666,71 @@ class MESICoherence extends CoherencePolicyWithUncached { val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) - def isHit (cmd: UInt, state: UInt): Bool = { - Mux(isWriteIntent(cmd), (state === tileExclusiveClean || state === tileExclusiveDirty), - (state === tileShared || state === tileExclusiveClean || state === tileExclusiveDirty)) + def isHit (cmd: UInt, m: ClientMetadata): Bool = { + Mux(isWriteIntent(cmd), (m.state === clientExclusiveClean || m.state === clientExclusiveDirty), + (m.state === clientShared || m.state === clientExclusiveClean || m.state === clientExclusiveDirty)) } - def isValid (state: UInt): Bool = { - state != tileInvalid + def isValid (m: ClientMetadata): Bool = { + m.state != clientInvalid } + def isHit (incoming: Acquire, m: MasterMetadata) = isValid(m) + def isValid (m: MasterMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && messageIsUncached(outstanding)) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) } - def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { - MuxLookup(cmd, (state === tileExclusiveDirty), Array( - M_INV -> (state === tileExclusiveDirty), - M_CLN -> (state === tileExclusiveDirty) + def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { + MuxLookup(cmd, (m.state === clientExclusiveDirty), Array( + M_INV -> (m.state === clientExclusiveDirty), + M_CLN -> (m.state === clientExclusiveDirty) )) } - def needsWriteback (state: UInt): Bool = { - needsTransactionOnCacheControl(M_INV, state) + def needsWriteback (m: ClientMetadata): Bool = { + needsTransactionOnCacheControl(M_INV, m) } - def newStateOnHit(cmd: UInt, state: UInt): UInt = { - Mux(isWrite(cmd), tileExclusiveDirty, state) - } - def newStateOnCacheControl(cmd: UInt) = { - MuxLookup(cmd, tileInvalid, Array( - M_INV -> tileInvalid, - M_CLN -> tileShared - )) - } - def newStateOnWriteback() = newStateOnCacheControl(M_INV) - def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt = { - MuxLookup(incoming.g_type, tileInvalid, Array( - grantReadShared -> tileShared, - grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusive, tileExclusiveDirty, tileExclusiveClean), - grantReadExclusiveAck -> tileExclusiveDirty, - grantReadUncached -> tileInvalid, - grantWriteUncached -> tileInvalid, - grantReadWordUncached -> tileInvalid, - grantWriteWordUncached -> tileInvalid, - grantAtomicUncached -> tileInvalid - )) - } - def newStateOnProbe(incoming: Probe, state: UInt): UInt = { - MuxLookup(incoming.p_type, state, Array( - probeInvalidate -> tileInvalid, - probeDowngrade -> tileShared, - probeCopy -> state + def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state))(this) + + def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( + MuxLookup(cmd, clientInvalid, Array( + M_INV -> clientInvalid, + M_CLN -> clientShared + )))(this) + def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( + MuxLookup(incoming.g_type, clientInvalid, Array( + grantReadShared -> clientShared, + grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusive, clientExclusiveDirty, clientExclusiveClean), + grantReadExclusiveAck -> clientExclusiveDirty, + grantReadUncached -> clientInvalid, + grantWriteUncached -> clientInvalid, + grantReadWordUncached -> clientInvalid, + grantWriteWordUncached -> clientInvalid, + grantAtomicUncached -> clientInvalid + )))(this) + def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( + MuxLookup(incoming.p_type, m.state, Array( + probeInvalidate -> clientInvalid, + probeDowngrade -> clientShared, + probeCopy -> m.state + )))(this) + def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) + def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt) = { + val popped = m.sharers.pop(src) + val next = MasterMetadata( + Mux(popped.none(), masterInvalid, + Mux(popped.one(), masterExclusive, masterShared)), popped)(this) + def is(r: UInt) = incoming.r_type === r + MuxBundle(m, Array( + is(releaseVoluntaryInvalidateData) -> next, + is(releaseInvalidateData) -> next, + is(releaseDowngradeData) -> m, + is(releaseCopyData) -> m, + is(releaseInvalidateAck) -> next, + is(releaseDowngradeAck) -> m, + is(releaseCopyAck) -> m )) } @@ -685,15 +743,15 @@ class MESICoherence extends CoherencePolicyWithUncached { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = { + def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) } - def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { + def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, outstanding.a_type) } def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { + def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -704,7 +762,7 @@ class MESICoherence extends CoherencePolicyWithUncached { probeDowngrade -> releaseDowngradeAck, probeCopy -> releaseCopyAck )) - Mux(needsWriteback(state), with_data, without_data) + Mux(needsWriteback(m), with_data, without_data) } def messageHasData(msg: SourcedMessage) = msg match { @@ -738,7 +796,7 @@ class MESICoherence extends CoherencePolicyWithUncached { } - def getProbeType(a_type: UInt, global_state: UInt): UInt = { + def getProbeType(a_type: UInt, m: MasterMetadata): UInt = { MuxLookup(a_type, probeCopy, Array( acquireReadShared -> probeDowngrade, acquireReadExclusive -> probeInvalidate, @@ -750,15 +808,14 @@ class MESICoherence extends CoherencePolicyWithUncached { )) } - def needsOuterRead(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterRead(a_type: UInt) = { (a_type != acquireWriteUncached) } - def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterWrite(a_type: UInt) = { (a_type === acquireWriteUncached) } - def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { - needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) - } + def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) + def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) @@ -767,15 +824,16 @@ class MESICoherence extends CoherencePolicyWithUncached { def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MigratoryCoherence extends CoherencePolicyWithUncached { +class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 7 - def nMasterStates = 0 + def nMasterStates = 3 def nAcquireTypes = 8 def nProbeTypes = 4 def nReleaseTypes = 11 def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: tileSharedByTwo :: tileMigratoryClean :: tileMigratoryDirty :: Nil = Enum(UInt(), nClientStates) + val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: clientSharedByTwo :: clientMigratoryClean :: clientMigratoryDirty :: Nil = Enum(UInt(), nClientStates) + val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(UInt(), nProbeTypes) @@ -787,72 +845,92 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadMigratory, grantReadWordUncached, grantAtomicUncached) val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) - def isHit (cmd: UInt, state: UInt): Bool = { - Mux(isWriteIntent(cmd), Vec(tileExclusiveClean, tileExclusiveDirty, tileMigratoryClean, tileMigratoryDirty).contains(state), (state != tileInvalid)) + def isHit (cmd: UInt, m: ClientMetadata): Bool = { + Mux(isWriteIntent(cmd), Vec(clientExclusiveClean, clientExclusiveDirty, clientMigratoryClean, clientMigratoryDirty).contains(m.state), (m.state != clientInvalid)) } - def isValid (state: UInt): Bool = { - state != tileInvalid + def isValid (m: ClientMetadata): Bool = { + m.state != clientInvalid } + def isHit (incoming: Acquire, m: MasterMetadata) = isValid(m) + def isValid (m: MasterMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && messageIsUncached(outstanding)) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive && outstanding.a_type != acquireInvalidateOthers)) } - def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { - MuxLookup(cmd, (state === tileExclusiveDirty), Array( - M_INV -> Vec(tileExclusiveDirty,tileMigratoryDirty).contains(state), - M_CLN -> Vec(tileExclusiveDirty,tileMigratoryDirty).contains(state) + def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { + MuxLookup(cmd, (m.state === clientExclusiveDirty), Array( + M_INV -> Vec(clientExclusiveDirty,clientMigratoryDirty).contains(m.state), + M_CLN -> Vec(clientExclusiveDirty,clientMigratoryDirty).contains(m.state) )) } - def needsWriteback (state: UInt): Bool = { - needsTransactionOnCacheControl(M_INV, state) + def needsWriteback (m: ClientMetadata): Bool = { + needsTransactionOnCacheControl(M_INV, m) } - def newStateOnHit(cmd: UInt, state: UInt): UInt = { - Mux(isWrite(cmd), MuxLookup(state, tileExclusiveDirty, Array( - tileExclusiveClean -> tileExclusiveDirty, - tileMigratoryClean -> tileMigratoryDirty)), state) - } - def newStateOnCacheControl(cmd: UInt) = { - MuxLookup(cmd, tileInvalid, Array( - M_INV -> tileInvalid, - M_CLN -> tileShared - )) - } - def newStateOnWriteback() = newStateOnCacheControl(M_INV) - def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt = { - MuxLookup(incoming.g_type, tileInvalid, Array( - grantReadShared -> tileShared, - grantReadExclusive -> MuxLookup(outstanding.a_type, tileExclusiveDirty, Array( - acquireReadExclusive -> tileExclusiveDirty, - acquireReadShared -> tileExclusiveClean)), - grantReadExclusiveAck -> tileExclusiveDirty, - grantReadUncached -> tileInvalid, - grantWriteUncached -> tileInvalid, - grantReadWordUncached -> tileInvalid, - grantWriteWordUncached -> tileInvalid, - grantAtomicUncached -> tileInvalid, - grantReadMigratory -> MuxLookup(outstanding.a_type, tileMigratoryDirty, Array( - acquireInvalidateOthers -> tileMigratoryDirty, - acquireReadExclusive -> tileMigratoryDirty, - acquireReadShared -> tileMigratoryClean)) - )) - } - def newStateOnProbe(incoming: Probe, state: UInt): UInt = { - MuxLookup(incoming.p_type, state, Array( - probeInvalidate -> tileInvalid, - probeInvalidateOthers -> tileInvalid, - probeCopy -> state, - probeDowngrade -> MuxLookup(state, tileShared, Array( - tileExclusiveClean -> tileSharedByTwo, - tileExclusiveDirty -> tileSharedByTwo, - tileSharedByTwo -> tileShared, - tileMigratoryClean -> tileSharedByTwo, - tileMigratoryDirty -> tileInvalid)) + def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata( + Mux(isWrite(cmd), MuxLookup(m.state, clientExclusiveDirty, Array( + clientExclusiveClean -> clientExclusiveDirty, + clientMigratoryClean -> clientMigratoryDirty)), m.state))(this) + def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( + MuxLookup(cmd, clientInvalid, Array( + M_INV -> clientInvalid, + M_CLN -> clientShared + )))(this) + def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( + MuxLookup(incoming.g_type, clientInvalid, Array( + grantReadShared -> clientShared, + grantReadExclusive -> MuxLookup(outstanding.a_type, clientExclusiveDirty, Array( + acquireReadExclusive -> clientExclusiveDirty, + acquireReadShared -> clientExclusiveClean)), + grantReadExclusiveAck -> clientExclusiveDirty, + grantReadUncached -> clientInvalid, + grantWriteUncached -> clientInvalid, + grantReadWordUncached -> clientInvalid, + grantWriteWordUncached -> clientInvalid, + grantAtomicUncached -> clientInvalid, + grantReadMigratory -> MuxLookup(outstanding.a_type, clientMigratoryDirty, Array( + acquireInvalidateOthers -> clientMigratoryDirty, + acquireReadExclusive -> clientMigratoryDirty, + acquireReadShared -> clientMigratoryClean)) + )))(this) + def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( + MuxLookup(incoming.p_type, m.state, Array( + probeInvalidate -> clientInvalid, + probeInvalidateOthers -> clientInvalid, + probeCopy -> m.state, + probeDowngrade -> MuxLookup(m.state, clientShared, Array( + clientExclusiveClean -> clientSharedByTwo, + clientExclusiveDirty -> clientSharedByTwo, + clientSharedByTwo -> clientShared, + clientMigratoryClean -> clientSharedByTwo, + clientMigratoryDirty -> clientInvalid)) + )))(this) + def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) + def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt) = { + val popped = m.sharers.pop(src) + val next = MasterMetadata( + Mux(popped.none(), masterInvalid, + Mux(popped.one(), masterExclusive, masterShared)), + popped)(this) + def is(r: UInt) = incoming.r_type === r + MuxBundle(m, Array( + is(releaseVoluntaryInvalidateData) -> next, + is(releaseInvalidateData) -> next, + is(releaseDowngradeData) -> m, + is(releaseCopyData) -> m, + is(releaseInvalidateAck) -> next, + is(releaseDowngradeAck) -> m, + is(releaseCopyAck) -> m, + is(releaseDowngradeDataMigratory) -> m, + is(releaseDowngradeAckHasCopy) -> m, + is(releaseInvalidateDataMigratory) -> next, + is(releaseInvalidateAckMigratory) -> next )) } + def getUncachedReadAcquireType = acquireReadUncached def getUncachedWriteAcquireType = acquireWriteUncached def getUncachedReadWordAcquireType = acquireReadWordUncached @@ -862,28 +940,28 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = { - Mux(isWriteIntent(cmd), Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), acquireReadShared) + def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { + Mux(isWriteIntent(cmd), Mux(m.state === clientInvalid, acquireReadExclusive, acquireInvalidateOthers), acquireReadShared) } - def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { - Mux(isWriteIntent(cmd), Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), outstanding.a_type) + def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = { + Mux(isWriteIntent(cmd), Mux(m.state === clientInvalid, acquireReadExclusive, acquireInvalidateOthers), outstanding.a_type) } def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { + def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> Mux(Vec(tileExclusiveDirty, tileMigratoryDirty).contains(state), + probeInvalidate -> Mux(Vec(clientExclusiveDirty, clientMigratoryDirty).contains(m.state), releaseInvalidateDataMigratory, releaseInvalidateData), - probeDowngrade -> Mux(state === tileMigratoryDirty, releaseDowngradeDataMigratory, releaseDowngradeData), + probeDowngrade -> Mux(m.state === clientMigratoryDirty, releaseDowngradeDataMigratory, releaseDowngradeData), probeCopy -> releaseCopyData )) val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( - probeInvalidate -> Mux(tileExclusiveClean === state, releaseInvalidateAckMigratory, releaseInvalidateAck), - probeInvalidateOthers -> Mux(state === tileSharedByTwo, releaseInvalidateAckMigratory, releaseInvalidateAck), - probeDowngrade -> Mux(state != tileInvalid, releaseDowngradeAckHasCopy, releaseDowngradeAck), + probeInvalidate -> Mux(clientExclusiveClean === m.state, releaseInvalidateAckMigratory, releaseInvalidateAck), + probeInvalidateOthers -> Mux(m.state === clientSharedByTwo, releaseInvalidateAckMigratory, releaseInvalidateAck), + probeDowngrade -> Mux(m.state != clientInvalid, releaseDowngradeAckHasCopy, releaseDowngradeAck), probeCopy -> releaseCopyAck )) - Mux(needsWriteback(state), with_data, without_data) + Mux(needsWriteback(m), with_data, without_data) } def messageHasData(msg: SourcedMessage) = msg match { @@ -906,7 +984,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { acquireReadWordUncached -> grantReadWordUncached, acquireWriteWordUncached -> grantWriteWordUncached, acquireAtomicUncached -> grantAtomicUncached, - acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI? + acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI? )) } def getGrantType(rel: Release, count: UInt): UInt = { @@ -916,7 +994,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } - def getProbeType(a_type: UInt, global_state: UInt): UInt = { + def getProbeType(a_type: UInt, m: MasterMetadata): UInt = { MuxLookup(a_type, probeCopy, Array( acquireReadShared -> probeDowngrade, acquireReadExclusive -> probeInvalidate, @@ -929,15 +1007,14 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { )) } - def needsOuterRead(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterRead(a_type: UInt) = { (a_type != acquireWriteUncached && a_type != acquireInvalidateOthers) } - def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterWrite(a_type: UInt) = { (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached || a_type === acquireAtomicUncached) } - def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { - needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) - } + def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) + def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index dea3e801..22fa933c 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -232,7 +232,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int)(implicit tlconf: TileLinkCon buf_out := io.uncached.acquire.bits.payload.data tag_out := io.uncached.acquire.bits.payload.client_xact_id addr_out := io.uncached.acquire.bits.payload.addr - has_data := tlconf.co.needsOuterWrite(io.uncached.acquire.bits.payload.a_type, UInt(0)) + has_data := tlconf.co.messageHasData(io.uncached.acquire.bits.payload) } when(active_out) { when(mem_cmd_q.io.enq.fire()) { diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index c8e33216..09bb0714 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -167,7 +167,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC val curr_p_id = PriorityEncoder(probe_flags) val pending_outer_write = co.messageHasData(xact) - val pending_outer_read = co.needsOuterRead(xact.a_type, UInt(0)) + val pending_outer_read = co.requiresOuterRead(xact.a_type) val outer_write_acq = Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId), xact.data) val outer_write_rel = Acquire(co.getUncachedWriteAcquireType, @@ -195,7 +195,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact.a_type, UInt(0)), + io.inner.probe.bits.payload := Probe(co.getProbeType(xact.a_type, co.masterMetadataOnFlush), xact.addr, UInt(trackerId)) @@ -215,7 +215,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC is(s_idle) { io.inner.acquire.ready := Bool(true) val needs_outer_write = co.messageHasData(c_acq.payload) - val needs_outer_read = co.needsOuterRead(c_acq.payload.a_type, UInt(0)) + val needs_outer_read = co.requiresOuterRead(c_acq.payload.a_type) when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src From 1ae7a9376cb2a0ec631eb1d634171f767327a20c Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 13 Jun 2014 03:25:52 -0700 Subject: [PATCH 229/374] Fix unhandled LLC writeback hazard --- uncore/src/main/scala/llc.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index 382a5f96..8da87042 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -438,10 +438,11 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, ta replay_s2_rdy := s3_rdy && !tag_we io.cpu.resp <> data.io.resp - io.cpu.req_cmd.ready := !s1_valid && !s2_valid && !replay_s2 && !tag_we io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready io.mem.req_cmd <> memCmdArb.io.out io.mem.req_data <> writeback.io.mem.req_data + io.cpu.req_cmd.ready := !(s1_valid || s2_valid || replay_s2 || tag_we || + io.cpu.req_cmd.bits.rw && io.cpu.req_data.ready) } class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module From a04ef4f5f42e8f5ed758d47c89656266b9ac12ad Mon Sep 17 00:00:00 2001 From: Jim Lawson Date: Fri, 13 Jun 2014 11:44:43 -0700 Subject: [PATCH 230/374] Quick change to work with new Width class. Replace .width with .needWidth() --- uncore/src/main/scala/htif.scala | 2 +- uncore/src/main/scala/llc.scala | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index cc1c1c62..28cb5217 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -91,7 +91,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti val cmd_readmem :: cmd_writemem :: cmd_readcr :: cmd_writecr :: cmd_ack :: cmd_nack :: Nil = Enum(UInt(), 6) - val pcr_addr = addr(io.cpu(0).pcr_req.bits.addr.width-1, 0) + val pcr_addr = addr(io.cpu(0).pcr_req.bits.addr.needWidth()-1, 0) val pcr_coreid = addr(log2Up(nTiles)-1+20+1,20) val pcr_wdata = packet_ram(0) diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index 8da87042..1c8671d0 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -15,8 +15,8 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn val rdata = gen.asOutput } val data = gen - val colMux = if (2*data.width <= leaf.data.width && n > leaf.n) 1 << math.floor(math.log(leaf.data.width/data.width)/math.log(2)).toInt else 1 - val nWide = if (data.width > leaf.data.width) 1+(data.width-1)/leaf.data.width else 1 + val colMux = if (2*data.needWidth() <= leaf.data.needWidth() && n > leaf.n) 1 << math.floor(math.log(leaf.data.needWidth()/data.needWidth())/math.log(2)).toInt else 1 + val nWide = if (data.needWidth() > leaf.data.needWidth()) 1+(data.needWidth()-1)/leaf.data.needWidth() else 1 val nDeep = if (n > colMux*leaf.n) 1+(n-1)/(colMux*leaf.n) else 1 if (nDeep > 1 || colMux > 1) require(isPow2(n) && isPow2(leaf.n)) @@ -39,10 +39,10 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn var dout: Bits = null val ridx = if (postLatency > 0) Reg(Bits()) else null - var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) + var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.needWidth()*(j+1))-1, leaf.data.needWidth()*j)) if (colMux > 1) - wmask0 = wmask0 & FillInterleaved(gen.width, UIntToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) - val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) + wmask0 = wmask0 & FillInterleaved(gen.needWidth(), UIntToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) + val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.needWidth()*(j+1))-1, leaf.data.needWidth()*j)) when (in.valid) { when (in.bits.rw) { mem.write(idx, wdata0, wmask0) } .otherwise { if (postLatency > 0) ridx := idx } @@ -61,7 +61,7 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn var colMuxOut = rdataWide if (colMux > 1) { - val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.width*(k+1)-1, gen.width*k))) + val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.needWidth()*(k+1)-1, gen.needWidth()*k))) colMuxOut = colMuxIn(r.bits(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) } @@ -268,7 +268,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[ data.io.in.bits.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUInt data.io.in.bits.rw := io.req.bits.rw data.io.in.bits.wdata := io.req_data.bits.data - data.io.in.bits.wmask := SInt(-1, io.req_data.bits.data.width) + data.io.in.bits.wmask := SInt(-1, io.req_data.bits.data.needWidth()) when (valid) { data.io.in.valid := Mux(req.rw, io.req_data.valid, qReady) data.io.in.bits.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUInt @@ -385,7 +385,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, ta for (i <- 0 until ways) s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) } - val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.width-1, s2.addr.width-tagWidth) === t(tagWidth-1, 0)) + val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.needWidth()-1, s2.addr.needWidth()-tagWidth) === t(tagWidth-1, 0)) val s2_hit_way = OHToUInt(s2_hits) val s2_hit = s2_hits.reduceLeft(_||_) val s2_hit_dirty = s2_tags(s2_hit_way)(tagWidth+1) @@ -395,7 +395,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, ta val tag_we = initialize || setDirty || mshr.io.tag.fire() val tag_waddr = Mux(initialize, initCount, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)) - val tag_wdata = Cat(setDirty, !initialize, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)) + val tag_wdata = Cat(setDirty, !initialize, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.needWidth()-1, mshr.io.tag.bits.addr.needWidth()-tagWidth)) val tag_wmask = Mux(initialize, SInt(-1, ways), UIntToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way))) tags.io.in.valid := io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy || tag_we tags.io.in.bits.addr := Mux(tag_we, tag_waddr, Mux(replay_s2, s2.addr, io.cpu.req_cmd.bits.addr)(log2Up(sets)-1,0)) From 0020ded36745bad05fa7376d08b828311f84e6cb Mon Sep 17 00:00:00 2001 From: Jim Lawson Date: Fri, 13 Jun 2014 14:53:48 -0700 Subject: [PATCH 231/374] Replace needWidth() with getWidth. --- uncore/src/main/scala/htif.scala | 2 +- uncore/src/main/scala/llc.scala | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 28cb5217..29eba5da 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -91,7 +91,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti val cmd_readmem :: cmd_writemem :: cmd_readcr :: cmd_writecr :: cmd_ack :: cmd_nack :: Nil = Enum(UInt(), 6) - val pcr_addr = addr(io.cpu(0).pcr_req.bits.addr.needWidth()-1, 0) + val pcr_addr = addr(io.cpu(0).pcr_req.bits.addr.getWidth-1, 0) val pcr_coreid = addr(log2Up(nTiles)-1+20+1,20) val pcr_wdata = packet_ram(0) diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index 1c8671d0..3a21cdca 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -15,8 +15,8 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn val rdata = gen.asOutput } val data = gen - val colMux = if (2*data.needWidth() <= leaf.data.needWidth() && n > leaf.n) 1 << math.floor(math.log(leaf.data.needWidth()/data.needWidth())/math.log(2)).toInt else 1 - val nWide = if (data.needWidth() > leaf.data.needWidth()) 1+(data.needWidth()-1)/leaf.data.needWidth() else 1 + val colMux = if (2*data.getWidth <= leaf.data.getWidth && n > leaf.n) 1 << math.floor(math.log(leaf.data.getWidth/data.getWidth)/math.log(2)).toInt else 1 + val nWide = if (data.getWidth > leaf.data.getWidth) 1+(data.getWidth-1)/leaf.data.getWidth else 1 val nDeep = if (n > colMux*leaf.n) 1+(n-1)/(colMux*leaf.n) else 1 if (nDeep > 1 || colMux > 1) require(isPow2(n) && isPow2(leaf.n)) @@ -39,10 +39,10 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn var dout: Bits = null val ridx = if (postLatency > 0) Reg(Bits()) else null - var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.needWidth()*(j+1))-1, leaf.data.needWidth()*j)) + var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.getWidth*(j+1))-1, leaf.data.getWidth*j)) if (colMux > 1) - wmask0 = wmask0 & FillInterleaved(gen.needWidth(), UIntToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) - val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.needWidth()*(j+1))-1, leaf.data.needWidth()*j)) + wmask0 = wmask0 & FillInterleaved(gen.getWidth, UIntToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) + val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.getWidth*(j+1))-1, leaf.data.getWidth*j)) when (in.valid) { when (in.bits.rw) { mem.write(idx, wdata0, wmask0) } .otherwise { if (postLatency > 0) ridx := idx } @@ -61,7 +61,7 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn var colMuxOut = rdataWide if (colMux > 1) { - val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.needWidth()*(k+1)-1, gen.needWidth()*k))) + val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.getWidth*(k+1)-1, gen.getWidth*k))) colMuxOut = colMuxIn(r.bits(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) } @@ -268,7 +268,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[ data.io.in.bits.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUInt data.io.in.bits.rw := io.req.bits.rw data.io.in.bits.wdata := io.req_data.bits.data - data.io.in.bits.wmask := SInt(-1, io.req_data.bits.data.needWidth()) + data.io.in.bits.wmask := SInt(-1, io.req_data.bits.data.getWidth) when (valid) { data.io.in.valid := Mux(req.rw, io.req_data.valid, qReady) data.io.in.bits.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUInt @@ -385,7 +385,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, ta for (i <- 0 until ways) s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) } - val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.needWidth()-1, s2.addr.needWidth()-tagWidth) === t(tagWidth-1, 0)) + val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.getWidth-1, s2.addr.getWidth-tagWidth) === t(tagWidth-1, 0)) val s2_hit_way = OHToUInt(s2_hits) val s2_hit = s2_hits.reduceLeft(_||_) val s2_hit_dirty = s2_tags(s2_hit_way)(tagWidth+1) @@ -395,7 +395,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, ta val tag_we = initialize || setDirty || mshr.io.tag.fire() val tag_waddr = Mux(initialize, initCount, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)) - val tag_wdata = Cat(setDirty, !initialize, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.needWidth()-1, mshr.io.tag.bits.addr.needWidth()-tagWidth)) + val tag_wdata = Cat(setDirty, !initialize, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.getWidth-1, mshr.io.tag.bits.addr.getWidth-tagWidth)) val tag_wmask = Mux(initialize, SInt(-1, ways), UIntToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way))) tags.io.in.valid := io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy || tag_we tags.io.in.bits.addr := Mux(tag_we, tag_waddr, Mux(replay_s2, s2.addr, io.cpu.req_cmd.bits.addr)(log2Up(sets)-1,0)) From f411fdcce39c354061348bd74968e86eaaeee100 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 8 Aug 2014 12:21:57 -0700 Subject: [PATCH 232/374] Full conversion to params. Compiles but does not elaborate. --- uncore/src/main/scala/cache.scala | 252 +++++++++----------------- uncore/src/main/scala/htif.scala | 54 +++--- uncore/src/main/scala/llc.scala | 37 ++-- uncore/src/main/scala/memserdes.scala | 94 +++++----- uncore/src/main/scala/network.scala | 57 +++--- uncore/src/main/scala/tilelink.scala | 106 ++++++----- uncore/src/main/scala/uncore.scala | 71 ++++---- 7 files changed, 301 insertions(+), 370 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 9a5afee3..247110a5 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -1,75 +1,18 @@ package uncore import Chisel._ -trait CacheConfig { - def sets: Int - def ways: Int - def tl: TileLinkConfiguration - def as: AddressSpaceConfiguration - def dm: Boolean - def states: Int - def lines: Int - def tagbits: Int - def idxbits: Int - def offbits: Int - def untagbits: Int - def rowbits: Int -} - -case class L2CacheConfig( - val sets: Int, val ways: Int, - val nrpq: Int, val nsdq: Int, - val nReleaseTransactions: Int, - val nAcquireTransactions: Int, - val tl: TileLinkConfiguration, - val as: AddressSpaceConfiguration) - extends CoherenceAgentConfiguration - with CacheConfig -{ - def states = tl.co.nMasterStates - def lines = sets*ways - def dm = ways == 1 - def offbits = 0 - def lineaddrbits = tl.addrBits - def idxbits = log2Up(sets) - def waybits = log2Up(ways) - def untagbits = offbits + idxbits - def tagbits = lineaddrbits - idxbits - def wordbits = 64 - def wordbytes = wordbits/8 - def wordoffbits = log2Up(wordbytes) - def rowbits = tl.dataBits - def rowbytes = rowbits/8 - def rowoffbits = log2Up(rowbytes) - def refillcycles = tl.dataBits/(rowbits) - def statebits = log2Up(states) - def nTSHRs = nReleaseTransactions + nAcquireTransactions - - require(states > 0) - require(isPow2(sets)) - require(isPow2(ways)) // TODO: relax this - require(refillcycles == 1) //TODO: relax this? -} - -abstract trait CacheBundle extends Bundle { - implicit val cacheconf: CacheConfig - override def clone = this.getClass.getConstructors.head.newInstance(cacheconf).asInstanceOf[this.type] -} - -abstract trait L2CacheBundle extends Bundle { - implicit val l2cacheconf: L2CacheConfig - override def clone = this.getClass.getConstructors.head.newInstance(l2cacheconf).asInstanceOf[this.type] -} - -trait HasL2Id extends L2CacheBundle { - val id = UInt(width = log2Up(l2cacheconf.nTSHRs)) -} - -trait HasL2InternalRequestState extends L2CacheBundle { - val tag_match = Bool() - val old_meta = new L2Metadata - val way_en = Bits(width = l2cacheconf.ways) -} +case object NSets extends Field[Int] +case object NWays extends Field[Int] +case object IsDM extends Field[Boolean] +case object TagBits extends Field[Int] +case object IdxBits extends Field[Int] +case object OffBits extends Field[Int] +case object UntagBits extends Field[Int] +case object WayBits extends Field[Int] +case object RowBits extends Field[Int] +case object WordBits extends Field[Int] +case object RefillCycles extends Field[Int] +case object Replacer extends Field[() => ReplacementPolicy] abstract class ReplacementPolicy { def way: UInt @@ -77,54 +20,53 @@ abstract class ReplacementPolicy { def hit: Unit } -class RandomReplacement(implicit val cacheconf: CacheConfig) extends ReplacementPolicy { +class RandomReplacement(ways: Int) extends ReplacementPolicy { private val replace = Bool() replace := Bool(false) val lfsr = LFSR16(replace) - def way = if(cacheconf.dm) UInt(0) else lfsr(log2Up(cacheconf.ways)-1,0) + def way = if(ways == 1) UInt(0) else lfsr(log2Up(ways)-1,0) def miss = replace := Bool(true) def hit = {} } -abstract class Metadata(implicit val cacheconf: CacheConfig) extends CacheBundle { - val tag = Bits(width = cacheconf.tagbits) +abstract class Metadata extends Bundle { + val tag = Bits(width = params(TagBits)) val coh: CoherenceMetadata } -class MetaReadReq(implicit val cacheconf: CacheConfig) extends CacheBundle { - val idx = Bits(width = cacheconf.idxbits) +class MetaReadReq extends Bundle { + val idx = Bits(width = params(IdxBits)) } -class MetaWriteReq[T <: Metadata](gen: T)(implicit conf: CacheConfig) extends MetaReadReq { - val way_en = Bits(width = conf.ways) +class MetaWriteReq[T <: Metadata](gen: T) extends MetaReadReq { + val way_en = Bits(width = params(WayBits)) val data = gen.clone - override def clone = new MetaWriteReq(gen)(conf).asInstanceOf[this.type] + override def clone = new MetaWriteReq(gen).asInstanceOf[this.type] } -class MetadataArray[T <: Metadata](makeRstVal: () => T)(implicit conf: CacheConfig) extends Module { - implicit val tl = conf.tl +class MetadataArray[T <: Metadata](makeRstVal: () => T) extends Module { val rstVal = makeRstVal() val io = new Bundle { val read = Decoupled(new MetaReadReq).flip val write = Decoupled(new MetaWriteReq(rstVal.clone)).flip - val resp = Vec.fill(conf.ways){rstVal.clone.asOutput} + val resp = Vec.fill(params(NWays)){rstVal.clone.asOutput} } val metabits = rstVal.getWidth - val rst_cnt = Reg(init=UInt(0, log2Up(conf.sets+1))) - val rst = rst_cnt < UInt(conf.sets) + val rst_cnt = Reg(init=UInt(0, log2Up(params(NSets)+1))) + val rst = rst_cnt < UInt(params(NSets)) val waddr = Mux(rst, rst_cnt, io.write.bits.idx) val wdata = Mux(rst, rstVal, io.write.bits.data).toBits val wmask = Mux(rst, SInt(-1), io.write.bits.way_en) when (rst) { rst_cnt := rst_cnt+UInt(1) } - val tag_arr = Mem(UInt(width = metabits*conf.ways), conf.sets, seqRead = true) + val tag_arr = Mem(UInt(width = metabits*params(NWays)), params(NSets), seqRead = true) when (rst || io.write.valid) { - tag_arr.write(waddr, Fill(conf.ways, wdata), FillInterleaved(metabits, wmask)) + tag_arr.write(waddr, Fill(params(NWays), wdata), FillInterleaved(metabits, wmask)) } val tags = tag_arr(RegEnable(io.read.bits.idx, io.read.valid)) - for (w <- 0 until conf.ways) { + for (w <- 0 until params(NWays)) { val m = tags(metabits*(w+1)-1, metabits*w) io.resp(w) := rstVal.clone.fromBits(m) } @@ -133,62 +75,63 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T)(implicit conf: CacheConf io.write.ready := !rst } +trait HasL2Id extends Bundle { + val id = UInt(width = log2Up(params(NTransactors))) +} + +trait HasL2InternalRequestState extends Bundle { + val tag_match = Bool() + val old_meta = new L2Metadata + val way_en = Bits(width = params(NWays)) +} + object L2Metadata { - def apply(tag: Bits, coh: MasterMetadata)(implicit conf: L2CacheConfig) = { + def apply(tag: Bits, coh: MasterMetadata) = { val meta = new L2Metadata meta.tag := tag meta.coh := coh meta } } -class L2Metadata(implicit val l2cacheconf: L2CacheConfig) extends Metadata - with L2CacheBundle { - val coh = l2cacheconf.tl.co.masterMetadataOnFlush.clone +class L2Metadata extends Metadata { + val coh = params(TLCoherence).masterMetadataOnFlush.clone } -/* -class L3Metadata(implicit conf: L3CacheConfig) extends Metadata - with L3CacheBundle { - val coh = MixedMetaData(conf.innerTL.co, conf.outerTL.co) ? -} -*/ - -class L2MetaReadReq(implicit val l2cacheconf: L2CacheConfig) extends MetaReadReq - with HasL2Id { - val tag = Bits(width = l2cacheconf.tagbits) +class L2MetaReadReq extends MetaReadReq with HasL2Id { + val tag = Bits(width = params(TagBits)) } -class L2MetaWriteReq(implicit val l2cacheconf: L2CacheConfig) extends MetaWriteReq[L2Metadata](new L2Metadata) +class L2MetaWriteReq extends MetaWriteReq[L2Metadata](new L2Metadata) with HasL2Id -class L2MetaResp(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle +class L2MetaResp extends Bundle with HasL2Id with HasL2InternalRequestState -class L2MetadataArray(implicit conf: L2CacheConfig) extends Module { - implicit val tl = conf.tl +class L2MetadataArray extends Module { + val (co, ways) = (params(TLCoherence), params(NWays)) val io = new Bundle { val read = Decoupled(new L2MetaReadReq).flip val write = Decoupled(new L2MetaWriteReq).flip val resp = Valid(new L2MetaResp) } - val meta = Module(new MetadataArray(() => L2Metadata(UInt(0), tl.co.masterMetadataOnFlush))) + val meta = Module(new MetadataArray(() => L2Metadata(UInt(0), co.masterMetadataOnFlush))) meta.io.read <> io.read meta.io.write <> io.write val s1_clk_en = Reg(next = io.read.fire()) val s1_tag = RegEnable(io.read.bits.tag, io.read.valid) val s1_id = RegEnable(io.read.bits.id, io.read.valid) - def wayMap[T <: Data](f: Int => T) = Vec((0 until conf.ways).map(f)) + def wayMap[T <: Data](f: Int => T) = Vec((0 until ways).map(f)) val s1_tag_eq_way = wayMap((w: Int) => meta.io.resp(w).tag === s1_tag) - val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && tl.co.isValid(meta.io.resp(w).coh)).toBits + val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && co.isValid(meta.io.resp(w).coh)).toBits val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) val s2_tag_match = s2_tag_match_way.orR val s2_hit_coh = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).coh, s1_clk_en))) //val s2_hit = s2_tag_match && tl.co.isHit(s2_req.cmd, s2_hit_state) && s2_hit_state === tl.co.newStateOnHit(s2_req.cmd, s2_hit_state) - val replacer = new RandomReplacement + val replacer = params(Replacer)() val s1_replaced_way_en = UIntToOH(replacer.way) val s2_replaced_way_en = UIntToOH(RegEnable(replacer.way, s1_clk_en)) val s2_repl_meta = Mux1H(s2_replaced_way_en, wayMap((w: Int) => @@ -203,23 +146,21 @@ class L2MetadataArray(implicit conf: L2CacheConfig) extends Module { io.resp.bits.way_en := Mux(s2_tag_match, s2_tag_match_way, s2_replaced_way_en) } -class L2DataReadReq(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle - with HasL2Id { - val way_en = Bits(width = l2cacheconf.ways) - val addr = Bits(width = l2cacheconf.tl.addrBits) +class L2DataReadReq extends Bundle with HasL2Id { + val way_en = Bits(width = params(NWays)) + val addr = Bits(width = params(TLAddrBits)) } -class L2DataWriteReq(implicit l2cacheconf: L2CacheConfig) extends L2DataReadReq { - val wmask = Bits(width = l2cacheconf.tl.writeMaskBits) - val data = Bits(width = l2cacheconf.tl.dataBits) +class L2DataWriteReq extends L2DataReadReq { + val wmask = Bits(width = params(TLWriteMaskBits)) + val data = Bits(width = params(TLDataBits)) } -class L2DataResp(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle - with HasL2Id { - val data = Bits(width = l2cacheconf.tl.dataBits) +class L2DataResp extends Bundle with HasL2Id { + val data = Bits(width = params(TLDataBits)) } -class L2DataArray(implicit conf: L2CacheConfig) extends Module { +class L2DataArray extends Module { val io = new Bundle { val read = Decoupled(new L2DataReadReq).flip val write = Decoupled(new L2DataWriteReq).flip @@ -228,9 +169,9 @@ class L2DataArray(implicit conf: L2CacheConfig) extends Module { val waddr = io.write.bits.addr val raddr = io.read.bits.addr - val wmask = FillInterleaved(conf.wordbits, io.write.bits.wmask) - val resp = (0 until conf.ways).map { w => - val array = Mem(Bits(width=conf.rowbits), conf.sets*conf.refillcycles, seqRead = true) + val wmask = FillInterleaved(params(WordBits), io.write.bits.wmask) + val resp = (0 until params(NWays)).map { w => + val array = Mem(Bits(width=params(RowBits)), params(NSets)*params(RefillCycles), seqRead = true) when (io.write.bits.way_en(w) && io.write.valid) { array.write(waddr, io.write.bits.data, wmask) } @@ -244,23 +185,11 @@ class L2DataArray(implicit conf: L2CacheConfig) extends Module { io.write.ready := Bool(true) } -class L2InternalAcquire(implicit val l2cacheconf: L2CacheConfig) extends Acquire()(l2cacheconf.tl) - with HasL2InternalRequestState +class L2HellaCache(bankId: Int) extends CoherenceAgent { -class L2InternalRelease(implicit val l2cacheconf: L2CacheConfig) extends Release()(l2cacheconf.tl) - with HasL2InternalRequestState - -class InternalTileLinkIO(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle { - implicit val (tl, ln) = (l2cacheconf.tl, l2cacheconf.tl.ln) - val acquire = new DecoupledIO(new LogicalNetworkIO(new L2InternalAcquire)) - val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip - val release = new DecoupledIO(new LogicalNetworkIO(new L2InternalRelease)) - val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip - val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) -} - -class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceAgent { - implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) + require(isPow2(params(NSets))) + require(isPow2(params(NWays))) + require(params(RefillCycles) == 1) val tshrfile = Module(new TSHRFile(bankId)) val meta = Module(new L2MetadataArray) @@ -278,12 +207,12 @@ class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceA } -class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { - implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) +class TSHRFile(bankId: Int) extends Module { + val (co, nClients) = (params(TLCoherence), params(NClients)) val io = new Bundle { val inner = (new TileLinkIO).flip val outer = new UncachedTileLinkIO - val incoherent = Vec.fill(ln.nClients){Bool()}.asInput + val incoherent = Vec.fill(nClients){Bool()}.asInput val meta_read = Decoupled(new L2MetaReadReq) val meta_write = Decoupled(new L2MetaWriteReq) val meta_resp = Valid(new L2MetaResp).flip @@ -305,10 +234,9 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { } // Create TSHRs for outstanding transactions - val nTrackers = conf.nReleaseTransactions + conf.nAcquireTransactions - val trackerList = (0 until conf.nReleaseTransactions).map { id => + val trackerList = (0 until params(NReleaseTransactors)).map { id => Module(new L2VoluntaryReleaseTracker(id, bankId)) - } ++ (conf.nReleaseTransactions until nTrackers).map { id => + } ++ (params(NReleaseTransactors) until params(NTransactors)).map { id => Module(new L2AcquireTracker(id, bankId)) } @@ -373,12 +301,12 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { } -abstract class L2XactTracker()(implicit conf: L2CacheConfig) extends Module { - implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) +abstract class L2XactTracker extends Module { + val (co, nClients) = (params(TLCoherence),params(NClients)) val io = new Bundle { val inner = (new TileLinkIO).flip val outer = new UncachedTileLinkIO - val tile_incoherent = Bits(INPUT, ln.nClients) + val tile_incoherent = Bits(INPUT, nClients) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) val meta_read = Decoupled(new L2MetaReadReq) @@ -397,11 +325,11 @@ abstract class L2XactTracker()(implicit conf: L2CacheConfig) extends Module { } -class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig) extends L2XactTracker()(conf) { +class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) val xact = Reg{ new Release } - val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) val new_meta = Reg(new L2Metadata) val incoming_rel = io.inner.release.bits @@ -460,16 +388,16 @@ Mux(co.messageHasData(xact), s_mem, s_ack) } } -class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig) extends L2XactTracker()(conf) { +class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) val state = Reg(init=s_idle) val xact = Reg{ new Acquire } - val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) //TODO: Will need id reg for merged release xacts - val init_sharer_cnt = Reg(init=UInt(0, width = log2Up(ln.nClients))) - val release_count = if (ln.nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(ln.nClients))) - val probe_flags = Reg(init=Bits(0, width = ln.nClients)) + val init_sharer_cnt = Reg(init=UInt(0, width = log2Up(nClients))) + val release_count = if(nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(nClients))) + val probe_flags = Reg(init=Bits(0, width = nClients)) val curr_p_id = PriorityEncoder(probe_flags) val pending_outer_write = co.messageHasData(xact) @@ -480,12 +408,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig xact.addr, UInt(trackerId), c_rel.payload.data) val outer_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) - val probe_initial_flags = Bits(width = ln.nClients) + val probe_initial_flags = Bits(width = nClients) probe_initial_flags := Bits(0) - if (ln.nClients > 1) { + if (nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence val probe_self = Bool(true) //co.needsSelfProbe(io.inner.acquire.bits.payload) - val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(ln.nClients)-1,0))) + val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } @@ -501,7 +429,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact.a_type, conf.tl.co.masterMetadataOnFlush), + io.inner.probe.bits.payload := Probe(co.getProbeType(xact.a_type, co.masterMetadataOnFlush), xact.addr, UInt(trackerId)) @@ -525,9 +453,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src - init_sharer_cnt := UInt(ln.nClients) // TODO: Broadcast only + init_sharer_cnt := UInt(nClients) // TODO: Broadcast only probe_flags := probe_initial_flags - if(ln.nClients > 1) { + if(nClients > 1) { release_count := PopCount(probe_initial_flags) state := Mux(probe_initial_flags.orR, s_probe, Mux(needs_outer_write, s_mem_write, @@ -550,7 +478,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig io.outer.acquire.bits.payload := outer_write_rel when(io.outer.acquire.ready) { io.inner.release.ready := Bool(true) - if(ln.nClients > 1) release_count := release_count - UInt(1) + if(nClients > 1) release_count := release_count - UInt(1) when(release_count === UInt(1)) { state := Mux(pending_outer_write, s_mem_write, Mux(pending_outer_read, s_mem_read, s_make_grant)) @@ -558,7 +486,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig } } .otherwise { io.inner.release.ready := Bool(true) - if(ln.nClients > 1) release_count := release_count - UInt(1) + if(nClients > 1) release_count := release_count - UInt(1) when(release_count === UInt(1)) { state := Mux(pending_outer_write, s_mem_write, Mux(pending_outer_read, s_mem_read, s_make_grant)) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index cc1c1c62..153bad3a 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -4,6 +4,11 @@ import Chisel._ import Node._ import uncore._ +case object HTIFWidth extends Field[Int] +case object HTIFNSCR extends Field[Int] +case object HTIFOffsetBits extends Field[Int] +case object HTIFNCores extends Field[Int] + class HostIO(val w: Int) extends Bundle { val clk = Bool(OUTPUT) @@ -20,43 +25,48 @@ class PCRReq extends Bundle val data = Bits(width = 64) } -class HTIFIO(ntiles: Int) extends Bundle +class HTIFIO extends Bundle { val reset = Bool(INPUT) - val id = UInt(INPUT, log2Up(ntiles)) + val id = UInt(INPUT, log2Up(params(HTIFNCores))) val pcr_req = Decoupled(new PCRReq).flip val pcr_rep = Decoupled(Bits(width = 64)) - val ipi_req = Decoupled(Bits(width = log2Up(ntiles))) + val ipi_req = Decoupled(Bits(width = log2Up(params(HTIFNCores)))) val ipi_rep = Decoupled(Bool()).flip val debug_stats_pcr = Bool(OUTPUT) // wired directly to stats register // expected to be used to quickly indicate to testbench to do logging b/c in 'interesting' work } -class SCRIO(n: Int) extends Bundle +class SCRIO extends Bundle { - val rdata = Vec.fill(n){Bits(INPUT, 64)} + val rdata = Vec.fill(params(HTIFNSCR)){Bits(INPUT, 64)} val wen = Bool(OUTPUT) - val waddr = UInt(OUTPUT, log2Up(n)) + val waddr = UInt(OUTPUT, log2Up(params(HTIFNSCR))) val wdata = Bits(OUTPUT, 64) } -class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: TileLinkConfiguration) extends Module +class HTIF(pcr_RESET: Int) extends Module { - implicit val (ln, co) = (conf.ln, conf.co) - val nTiles = ln.nClients-1 // This HTIF is itself a TileLink client + val dataBits = params(TLDataBits) + val co = params(TLCoherence) + val w = params(HTIFWidth) + val nSCR = params(HTIFNSCR) + val offsetBits = params(HTIFOffsetBits) + val nCores = params(HTIFNCores) val io = new Bundle { val host = new HostIO(w) - val cpu = Vec.fill(nTiles){new HTIFIO(nTiles).flip} + val cpu = Vec.fill(nCores){new HTIFIO().flip} val mem = new TileLinkIO - val scr = new SCRIO(nSCR) + val scr = new SCRIO } + io.host.debug_stats_pcr := io.cpu.map(_.debug_stats_pcr).reduce(_||_) // system is 'interesting' if any tile is 'interesting' val short_request_bits = 64 - val long_request_bits = short_request_bits + conf.dataBits + val long_request_bits = short_request_bits + dataBits require(short_request_bits % w == 0) val rx_count_w = 13 + log2Up(64) - log2Up(w) // data size field is 12 bits @@ -92,7 +102,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti val cmd_readmem :: cmd_writemem :: cmd_readcr :: cmd_writecr :: cmd_ack :: cmd_nack :: Nil = Enum(UInt(), 6) val pcr_addr = addr(io.cpu(0).pcr_req.bits.addr.width-1, 0) - val pcr_coreid = addr(log2Up(nTiles)-1+20+1,20) + val pcr_coreid = addr(log2Up(nCores)-1+20+1,20) val pcr_wdata = packet_ram(0) val bad_mem_packet = size(offsetBits-1-3,0).orR || addr(offsetBits-1-3,0).orR @@ -114,13 +124,13 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti val mem_acked = Reg(init=Bool(false)) val mem_gxid = Reg(Bits()) - val mem_gsrc = Reg(UInt(width = conf.ln.idBits)) + val mem_gsrc = Reg(UInt()) val mem_needs_ack = Reg(Bool()) when (io.mem.grant.valid) { mem_acked := Bool(true) mem_gxid := io.mem.grant.bits.payload.master_xact_id mem_gsrc := io.mem.grant.bits.header.src - mem_needs_ack := conf.co.requiresAckForGrant(io.mem.grant.bits.payload.g_type) + mem_needs_ack := co.requiresAckForGrant(io.mem.grant.bits.payload.g_type) } io.mem.grant.ready := Bool(true) @@ -168,8 +178,8 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti } var mem_req_data: Bits = null - for (i <- 0 until conf.dataBits/short_request_bits) { - val idx = UInt(i, log2Up(conf.dataBits/short_request_bits)) + for (i <- 0 until dataBits/short_request_bits) { + val idx = UInt(i, log2Up(dataBits/short_request_bits)) when (state === state_mem_rresp && io.mem.grant.valid) { packet_ram(idx) := io.mem.grant.bits.payload.data((i+1)*short_request_bits-1, i*short_request_bits) } @@ -184,7 +194,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti acq_q.io.deq.ready := io.mem.acquire.ready io.mem.acquire.bits.payload := acq_q.io.deq.bits io.mem.acquire.bits.payload.data := mem_req_data - io.mem.acquire.bits.header.src := UInt(conf.ln.nClients) // By convention HTIF is the client with the largest id + io.mem.acquire.bits.header.src := UInt(params(LNClients)) // By convention HTIF is the client with the largest id io.mem.acquire.bits.header.dst := UInt(0) // DNC; Overwritten outside module io.mem.finish.valid := (state === state_mem_finish) && mem_needs_ack io.mem.finish.bits.payload.master_xact_id := mem_gxid @@ -194,7 +204,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti val pcr_reset = UInt(pcr_RESET)(pcr_addr.getWidth-1,0) val pcrReadData = Reg(Bits(width = io.cpu(0).pcr_rep.bits.getWidth)) - for (i <- 0 until nTiles) { + for (i <- 0 until nCores) { val my_reset = Reg(init=Bool(true)) val my_ipi = Reg(init=Bool(false)) @@ -211,7 +221,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti } cpu.ipi_rep.valid := my_ipi cpu.ipi_req.ready := Bool(true) - for (j <- 0 until nTiles) { + for (j <- 0 until nCores) { when (io.cpu(j).ipi_req.valid && io.cpu(j).ipi_req.bits === UInt(i)) { my_ipi := Bool(true) } @@ -239,8 +249,8 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti val scr_rdata = Vec.fill(io.scr.rdata.size){Bits(width = 64)} for (i <- 0 until scr_rdata.size) scr_rdata(i) := io.scr.rdata(i) - scr_rdata(0) := UInt(nTiles) - scr_rdata(1) := UInt((BigInt(conf.dataBits/8) << acq_q.io.enq.bits.addr.getWidth) >> 20) + scr_rdata(0) := UInt(nCores) + scr_rdata(1) := UInt((BigInt(dataBits/8) << acq_q.io.enq.bits.addr.getWidth) >> 20) io.scr.wen := Bool(false) io.scr.wdata := pcr_wdata diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index 382a5f96..5bc89e38 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -72,26 +72,26 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn io.rdata := Mux1H(rdataSel, rdataDeep) } -class LLCDataReq(ways: Int)(implicit conf: MemoryIFConfiguration) extends MemReqCmd +class LLCDataReq(ways: Int) extends MemReqCmd { val way = UInt(width = log2Up(ways)) val isWriteback = Bool() - override def clone = new LLCDataReq(ways)(conf).asInstanceOf[this.type] + override def clone = new LLCDataReq(ways).asInstanceOf[this.type] } -class LLCTagReq(ways: Int)(implicit val conf: MemoryIFConfiguration) extends HasMemAddr +class LLCTagReq(ways: Int) extends HasMemAddr { val way = UInt(width = log2Up(ways)) - override def clone = new LLCTagReq(ways)(conf).asInstanceOf[this.type] + override def clone = new LLCTagReq(ways).asInstanceOf[this.type] } -class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int)(implicit conf: MemoryIFConfiguration) extends Module +class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int) extends Module { val io = new Bundle { val cpu = Decoupled(new MemReqCmd).flip val repl_way = UInt(INPUT, log2Up(ways)) val repl_dirty = Bool(INPUT) - val repl_tag = UInt(INPUT, conf.addrBits - log2Up(sets)) + val repl_tag = UInt(INPUT, params(MIFAddrBits) - log2Up(sets)) val data = Decoupled(new LLCDataReq(ways)) val tag = Decoupled(new LLCTagReq(ways)) val mem = new MemPipeIO @@ -100,14 +100,14 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int)(im } class MSHR extends Bundle { - val addr = UInt(width = conf.addrBits) + val addr = UInt(width = params(MIFAddrBits)) val way = UInt(width = log2Up(ways)) val tag = io.cpu.bits.tag.clone val refilled = Bool() val refillCount = UInt(width = log2Up(refill_cycles)) val requested = Bool() val old_dirty = Bool() - val old_tag = UInt(width = conf.addrBits - log2Up(sets)) + val old_tag = UInt(width = params(MIFAddrBits) - log2Up(sets)) val wb_busy = Bool() override def clone = new MSHR().asInstanceOf[this.type] @@ -183,10 +183,10 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int)(im io.mem_resp_way := mshr(refillId).way } -class LLCWriteback(requestors: Int, refill_cycles: Int)(implicit conf: MemoryIFConfiguration) extends Module +class LLCWriteback(requestors: Int, refill_cycles: Int) extends Module { val io = new Bundle { - val req = Vec.fill(requestors){Decoupled(UInt(width = conf.addrBits)).flip } + val req = Vec.fill(requestors){Decoupled(UInt(width = params(MIFAddrBits))).flip } val data = Vec.fill(requestors){Decoupled(new MemData).flip } val mem = new MemPipeIO } @@ -232,12 +232,12 @@ class LLCWriteback(requestors: Int, refill_cycles: Int)(implicit conf: MemoryIFC io.mem.req_data.bits := io.data(who).bits } -class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[UInt])(implicit conf: MemoryIFConfiguration) extends Module +class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[UInt]) extends Module { val io = new Bundle { val req = Decoupled(new LLCDataReq(ways)).flip val req_data = Decoupled(new MemData).flip - val writeback = Decoupled(UInt(width = conf.addrBits)) + val writeback = Decoupled(UInt(width = params(MIFAddrBits))) val writeback_data = Decoupled(new MemData) val resp = Decoupled(new MemResp) val mem_resp = Valid(new MemResp).flip @@ -245,7 +245,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[ val mem_resp_way = UInt(INPUT, log2Up(ways)) } - val data = Module(new BigMem(sets*ways*refill_cycles, 1, latency-1, leaf)(Bits(width = conf.dataBits))) + val data = Module(new BigMem(sets*ways*refill_cycles, 1, latency-1, leaf)(Bits(width = params(MIFDataBits)))) class QEntry extends MemResp { val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] @@ -300,7 +300,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[ io.writeback_data.bits := q.io.deq.bits } -class MemReqArb(n: Int, refill_cycles: Int)(implicit conf: MemoryIFConfiguration) extends Module +class MemReqArb(n: Int, refill_cycles: Int) extends Module { val io = new Bundle { val cpu = Vec.fill(n){new MemIO().flip} @@ -343,16 +343,16 @@ class MemReqArb(n: Int, refill_cycles: Int)(implicit conf: MemoryIFConfiguration when (unlock) { lock := Bool(false) } } -abstract class DRAMSideLLCLike(implicit conf: MemoryIFConfiguration) extends Module { +abstract class DRAMSideLLCLike extends Module { val io = new Bundle { val cpu = new MemIO().flip val mem = new MemPipeIO } } -class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, tagLeaf: Mem[UInt], dataLeaf: Mem[UInt])(implicit conf: MemoryIFConfiguration) extends DRAMSideLLCLike +class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, tagLeaf: Mem[UInt], dataLeaf: Mem[UInt]) extends DRAMSideLLCLike { - val tagWidth = conf.addrBits - log2Up(sets) + val tagWidth = params(MIFAddrBits) - log2Up(sets) val metaWidth = tagWidth + 2 // valid + dirty val memCmdArb = Module(new Arbiter(new MemReqCmd, 2)) @@ -499,8 +499,7 @@ object HellaQueue } } -class DRAMSideLLCNull(numRequests: Int, refillCycles: Int)(implicit conf: MemoryIFConfiguration) - extends DRAMSideLLCLike { +class DRAMSideLLCNull(numRequests: Int, refillCycles: Int) extends DRAMSideLLCLike { val numEntries = numRequests * refillCycles val size = log2Down(numEntries) + 1 diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 22fa933c..eba971fd 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -2,57 +2,58 @@ package uncore import Chisel._ import scala.math._ -case class AddressSpaceConfiguration(paddrBits: Int, vaddrBits: Int, pgIdxBits: Int, asidBits: Int, permBits:Int) { - val ppnBits = paddrBits - pgIdxBits - val vpnBits = vaddrBits - pgIdxBits +case object PAddrBits extends Field[Int] +case object VAddrBits extends Field[Int] +case object PgIdxBits extends Field[Int] +case object ASIdBits extends Field[Int] +case object PermBits extends Field[Int] +case object PPNBits extends Field[Int] +case object VPNBits extends Field[Int] + +case object MIFAddrBits extends Field[Int] +case object MIFDataBits extends Field[Int] +case object MIFTagBits extends Field[Int] +case object MIFDataBeats extends Field[Int] + +trait HasMemData extends Bundle { + val data = Bits(width = params(MIFDataBits)) } -case class MemoryIFConfiguration(addrBits: Int, dataBits: Int, tagBits: Int, dataBeats: Int) - -abstract trait MemoryIFSubBundle extends Bundle { - implicit val conf: MemoryIFConfiguration - override def clone = this.getClass.getConstructors.head.newInstance(conf).asInstanceOf[this.type] +trait HasMemAddr extends Bundle { + val addr = UInt(width = params(MIFAddrBits)) } -trait HasMemData extends MemoryIFSubBundle { - val data = Bits(width = conf.dataBits) +trait HasMemTag extends Bundle { + val tag = UInt(width = params(MIFTagBits)) } -trait HasMemAddr extends MemoryIFSubBundle { - val addr = UInt(width = conf.addrBits) -} - -trait HasMemTag extends MemoryIFSubBundle { - val tag = UInt(width = conf.tagBits) -} - -class MemReqCmd(implicit val conf: MemoryIFConfiguration) extends HasMemAddr with HasMemTag { +class MemReqCmd extends HasMemAddr with HasMemTag { val rw = Bool() } -class MemResp(implicit val conf: MemoryIFConfiguration) extends HasMemData with HasMemTag +class MemResp extends HasMemData with HasMemTag -class MemData(implicit val conf: MemoryIFConfiguration) extends HasMemData +class MemData extends HasMemData -class MemIO(implicit val conf: MemoryIFConfiguration) extends Bundle { +class MemIO extends Bundle { val req_cmd = Decoupled(new MemReqCmd) val req_data = Decoupled(new MemData) val resp = Decoupled(new MemResp).flip } -class MemPipeIO(implicit val conf: MemoryIFConfiguration) extends Bundle { +class MemPipeIO extends Bundle { val req_cmd = Decoupled(new MemReqCmd) val req_data = Decoupled(new MemData) val resp = Valid(new MemResp).flip } -class MemSerializedIO(w: Int)(implicit val conf: MemoryIFConfiguration) extends Bundle +class MemSerializedIO(w: Int) extends Bundle { val req = Decoupled(Bits(width = w)) val resp = Valid(Bits(width = w)).flip } -class MemSerdes(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module +class MemSerdes(w: Int) extends Module { val io = new Bundle { val wide = new MemIO().flip @@ -61,6 +62,7 @@ class MemSerdes(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module val abits = io.wide.req_cmd.bits.toBits.getWidth val dbits = io.wide.req_data.bits.toBits.getWidth val rbits = io.wide.resp.bits.getWidth + val dbeats = params(MIFDataBeats) val out_buf = Reg(Bits()) val in_buf = Reg(Bits()) @@ -68,7 +70,7 @@ class MemSerdes(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(UInt(), 5) val state = Reg(init=s_idle) val send_cnt = Reg(init=UInt(0, log2Up((max(abits, dbits)+w-1)/w))) - val data_send_cnt = Reg(init=UInt(0, log2Up(conf.dataBeats))) + val data_send_cnt = Reg(init=UInt(0, log2Up(dbeats))) val adone = io.narrow.req.ready && send_cnt === UInt((abits-1)/w) val ddone = io.narrow.req.ready && send_cnt === UInt((dbits-1)/w) @@ -104,12 +106,12 @@ class MemSerdes(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module } when (state === s_write_data && ddone) { data_send_cnt := data_send_cnt + UInt(1) - state := Mux(data_send_cnt === UInt(conf.dataBeats-1), s_idle, s_write_idle) + state := Mux(data_send_cnt === UInt(dbeats-1), s_idle, s_write_idle) send_cnt := UInt(0) } val recv_cnt = Reg(init=UInt(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = Reg(init=UInt(0, log2Up(conf.dataBeats))) + val data_recv_cnt = Reg(init=UInt(0, log2Up(dbeats))) val resp_val = Reg(init=Bool(false)) resp_val := Bool(false) @@ -127,21 +129,22 @@ class MemSerdes(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module io.wide.resp.bits := io.wide.resp.bits.fromBits(in_buf) } -class MemDesserIO(w: Int)(implicit val conf: MemoryIFConfiguration) extends Bundle { +class MemDesserIO(w: Int) extends Bundle { val narrow = new MemSerializedIO(w).flip val wide = new MemIO } -class MemDesser(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module // test rig side +class MemDesser(w: Int) extends Module // test rig side { val io = new MemDesserIO(w) val abits = io.wide.req_cmd.bits.toBits.getWidth val dbits = io.wide.req_data.bits.toBits.getWidth val rbits = io.wide.resp.bits.getWidth + val dbeats = params(MIFDataBeats) require(dbits >= abits && rbits >= dbits) val recv_cnt = Reg(init=UInt(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = Reg(init=UInt(0, log2Up(conf.dataBeats))) + val data_recv_cnt = Reg(init=UInt(0, log2Up(dbeats))) val adone = io.narrow.req.valid && recv_cnt === UInt((abits-1)/w) val ddone = io.narrow.req.valid && recv_cnt === UInt((dbits-1)/w) val rdone = io.narrow.resp.valid && recv_cnt === UInt((rbits-1)/w) @@ -169,13 +172,13 @@ class MemDesser(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module } when (state === s_data && io.wide.req_data.ready) { state := s_data_recv - when (data_recv_cnt === UInt(conf.dataBeats-1)) { + when (data_recv_cnt === UInt(dbeats-1)) { state := s_cmd_recv } data_recv_cnt := data_recv_cnt + UInt(1) } when (rdone) { // state === s_reply - when (data_recv_cnt === UInt(conf.dataBeats-1)) { + when (data_recv_cnt === UInt(dbeats-1)) { state := s_cmd_recv } recv_cnt := UInt(0) @@ -189,7 +192,7 @@ class MemDesser(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module io.wide.req_data.valid := state === s_data io.wide.req_data.bits.data := in_buf >> UInt(((rbits+w-1)/w - (dbits+w-1)/w)*w) - val dataq = Module(new Queue(new MemResp, conf.dataBeats)) + val dataq = Module(new Queue(new MemResp, dbeats)) dataq.io.enq <> io.wide.resp dataq.io.deq.ready := recv_cnt === UInt((rbits-1)/w) @@ -198,18 +201,21 @@ class MemDesser(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module } //Adapter betweewn an UncachedTileLinkIO and a mem controller MemIO -class MemIOUncachedTileLinkIOConverter(qDepth: Int)(implicit tlconf: TileLinkConfiguration, mifconf: MemoryIFConfiguration) extends Module { +class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { val io = new Bundle { val uncached = new UncachedTileLinkIO().flip val mem = new MemIO } - - require(tlconf.dataBits == mifconf.dataBits*mifconf.dataBeats) - //require(tlconf.clientXactIdBits <= mifconf.tagBits) + val co = params(TLCoherence) + val tbits = params(MIFTagBits) + val dbits = params(MIFDataBits) + val dbeats = params(MIFDataBits) + require(params(TLDataBits) == dbits*dbeats) + //require(params(TLClientXactIdBits) <= params(MIFTagBits)) val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth)) val mem_data_q = Module(new Queue(new MemData, qDepth)) - val cnt_max = mifconf.dataBeats + val cnt_max = dbeats val cnt_out = Reg(UInt(width = log2Up(cnt_max+1))) val active_out = Reg(init=Bool(false)) @@ -222,7 +228,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int)(implicit tlconf: TileLinkCon val cnt_in = Reg(UInt(width = log2Up(cnt_max+1))) val active_in = Reg(init=Bool(false)) val buf_in = Reg(Bits()) - val tag_in = Reg(UInt(width = mifconf.tagBits)) + val tag_in = Reg(UInt(width = tbits)) // Decompose outgoing TL Acquires into MemIO cmd and data when(!active_out && io.uncached.acquire.valid) { @@ -232,7 +238,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int)(implicit tlconf: TileLinkCon buf_out := io.uncached.acquire.bits.payload.data tag_out := io.uncached.acquire.bits.payload.client_xact_id addr_out := io.uncached.acquire.bits.payload.addr - has_data := tlconf.co.messageHasData(io.uncached.acquire.bits.payload) + has_data := co.messageHasData(io.uncached.acquire.bits.payload) } when(active_out) { when(mem_cmd_q.io.enq.fire()) { @@ -240,7 +246,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int)(implicit tlconf: TileLinkCon } when(mem_data_q.io.enq.fire()) { cnt_out := cnt_out + UInt(1) - buf_out := buf_out >> UInt(mifconf.dataBits) + buf_out := buf_out >> UInt(dbits) } when(cmd_sent_out && (!has_data || cnt_out === UInt(cnt_max))) { active_out := Bool(false) @@ -264,7 +270,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int)(implicit tlconf: TileLinkCon when(!active_in && io.mem.resp.valid) { active_in := Bool(true) cnt_in := UInt(1) - buf_in := io.mem.resp.bits.data << UInt(mifconf.dataBits*(cnt_max-1)) + buf_in := io.mem.resp.bits.data << UInt(dbits*(cnt_max-1)) tag_in := io.mem.resp.bits.tag } when(active_in) { @@ -272,7 +278,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int)(implicit tlconf: TileLinkCon active_in := Bool(false) } when(io.mem.resp.fire()) { - buf_in := Cat(io.mem.resp.bits.data, buf_in(cnt_max*mifconf.dataBits-1,mifconf.dataBits)) + buf_in := Cat(io.mem.resp.bits.data, buf_in(cnt_max*dbits-1,dbits)) cnt_in := cnt_in + UInt(1) } } diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index 6889bd7f..72002d45 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -1,32 +1,35 @@ package uncore import Chisel._ -case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) +case object LNMasters extends Field[Int] +case object LNClients extends Field[Int] +case object LNEndpoints extends Field[Int] -class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { - val src = UInt(width = conf.idBits) - val dst = UInt(width = conf.idBits) +class PhysicalHeader(n: Int) extends Bundle { + val src = UInt(width = log2Up(n)) + val dst = UInt(width = log2Up(n)) } -class PhysicalNetworkIO[T <: Data](dType: T)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { - val header = new PhysicalHeader +class PhysicalNetworkIO[T <: Data](n: Int, dType: T) extends Bundle { + val header = new PhysicalHeader(n) val payload = dType.clone - override def clone = { new PhysicalNetworkIO(dType).asInstanceOf[this.type] } + override def clone = new PhysicalNetworkIO(n,dType).asInstanceOf[this.type] } -abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Module - -class BasicCrossbarIO[T <: Data](dType: T)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { - val in = Vec.fill(conf.nEndpoints){Decoupled(new PhysicalNetworkIO(dType))}.flip - val out = Vec.fill(conf.nEndpoints){Decoupled(new PhysicalNetworkIO(dType))} +class BasicCrossbarIO[T <: Data](n: Int, dType: T) extends Bundle { + val in = Vec.fill(n){Decoupled(new PhysicalNetworkIO(n,dType))}.flip + val out = Vec.fill(n){Decoupled(new PhysicalNetworkIO(n,dType))} } -class BasicCrossbar[T <: Data](dType: T, count: Int = 1)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { - val io = new BasicCrossbarIO(dType) - val rdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints)(Bool())) +abstract class PhysicalNetwork extends Module + +class BasicCrossbar[T <: Data](n: Int, dType: T, count: Int = 1) extends PhysicalNetwork { + val io = new BasicCrossbarIO(n, dType) + + val rdyVecs = List.fill(n){Vec.fill(n)(Bool())} io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { - val rrarb = Module(new LockingRRArbiter(io.in(0).bits, conf.nEndpoints, count)) + val rrarb = Module(new LockingRRArbiter(io.in(0).bits, n, count)) (rrarb.io.in, io.in, rdys).zipped.map{ case (arb, in, rdy) => { arb.valid := in.valid && (in.bits.header.dst === UInt(i)) arb.bits := in.bits @@ -34,30 +37,26 @@ class BasicCrossbar[T <: Data](dType: T, count: Int = 1)(implicit conf: Physical }} out <> rrarb.io.out }} - for(i <- 0 until conf.nEndpoints) { + for(i <- 0 until n) { io.in(i).ready := rdyVecs.map(r => r(i)).reduceLeft(_||_) } } -case class LogicalNetworkConfiguration(idBits: Int, nMasters: Int, nClients: Int) { - val nEndpoints = nMasters + nClients +abstract class LogicalNetwork extends Module + +class LogicalHeader extends Bundle { + val src = UInt(width = log2Up(params(LNEndpoints))) + val dst = UInt(width = log2Up(params(LNEndpoints))) } -abstract class LogicalNetwork[TileLinkType <: Bundle](implicit conf: LogicalNetworkConfiguration) extends Module - -class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val src = UInt(width = conf.idBits) - val dst = UInt(width = conf.idBits) -} - -class LogicalNetworkIO[T <: Data](dType: T)(implicit conf: LogicalNetworkConfiguration) extends Bundle { +class LogicalNetworkIO[T <: Data](dType: T) extends Bundle { val header = new LogicalHeader val payload = dType.clone override def clone = { new LogicalNetworkIO(dType).asInstanceOf[this.type] } } object DecoupledLogicalNetworkIOWrapper { - def apply[T <: Data](in: DecoupledIO[T], src: UInt = UInt(0), dst: UInt = UInt(0))(implicit conf: LogicalNetworkConfiguration) = { + def apply[T <: Data](in: DecoupledIO[T], src: UInt = UInt(0), dst: UInt = UInt(0)) = { val out = Decoupled(new LogicalNetworkIO(in.bits.clone)).asDirectionless out.valid := in.valid out.bits.payload := in.bits @@ -69,7 +68,7 @@ object DecoupledLogicalNetworkIOWrapper { } object DecoupledLogicalNetworkIOUnwrapper { - def apply[T <: Data](in: DecoupledIO[LogicalNetworkIO[T]])(implicit conf: LogicalNetworkConfiguration) = { + def apply[T <: Data](in: DecoupledIO[LogicalNetworkIO[T]]) = { val out = Decoupled(in.bits.payload.clone).asDirectionless out.valid := in.valid out.bits := in.bits.payload diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 68da8213..d49e264b 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1,27 +1,29 @@ package uncore import Chisel._ -case class TileLinkConfiguration(co: CoherencePolicyWithUncached, ln: LogicalNetworkConfiguration, addrBits: Int, masterXactIdBits: Int, clientXactIdBits: Int, dataBits: Int, writeMaskBits: Int, wordAddrBits: Int, atomicOpBits: Int) +case object TLCoherence extends Field[CoherencePolicyWithUncached] +case object TLAddrBits extends Field[Int] +case object TLMasterXactIdBits extends Field[Int] +case object TLClientXactIdBits extends Field[Int] +case object TLDataBits extends Field[Int] +case object TLWriteMaskBits extends Field[Int] +case object TLWordAddrBits extends Field[Int] +case object TLAtomicOpBits extends Field[Int] -abstract trait TileLinkSubBundle extends Bundle { - implicit val tlconf: TileLinkConfiguration - override def clone = this.getClass.getConstructors.head.newInstance(tlconf).asInstanceOf[this.type] +trait HasPhysicalAddress extends Bundle { + val addr = UInt(width = params(TLAddrBits)) } -trait HasPhysicalAddress extends TileLinkSubBundle { - val addr = UInt(width = tlconf.addrBits) +trait HasClientTransactionId extends Bundle { + val client_xact_id = Bits(width = params(TLClientXactIdBits)) } -trait HasClientTransactionId extends TileLinkSubBundle { - val client_xact_id = Bits(width = tlconf.clientXactIdBits) +trait HasMasterTransactionId extends Bundle { + val master_xact_id = Bits(width = params(TLMasterXactIdBits)) } -trait HasMasterTransactionId extends TileLinkSubBundle { - val master_xact_id = Bits(width = tlconf.masterXactIdBits) -} - -trait HasTileLinkData extends TileLinkSubBundle { - val data = Bits(width = tlconf.dataBits) +trait HasTileLinkData extends Bundle { + val data = Bits(width = params(TLDataBits)) } trait SourcedMessage extends Bundle @@ -30,7 +32,7 @@ trait MasterSourcedMessage extends SourcedMessage object Acquire { - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt)(implicit conf: TileLinkConfiguration): Acquire = { + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt): Acquire = { val acq = new Acquire acq.a_type := a_type acq.addr := addr @@ -41,42 +43,42 @@ object Acquire acq.atomic_opcode := Bits(0) acq } - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt)(implicit conf: TileLinkConfiguration): Acquire = { + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt): Acquire = { val acq = apply(a_type, addr, client_xact_id) acq.data := data acq } - def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, write_mask: Bits, data: UInt)(implicit conf: TileLinkConfiguration): Acquire = { + def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, write_mask: Bits, data: UInt): Acquire = { val acq = apply(a_type, addr, client_xact_id, data) acq.write_mask := write_mask acq } - def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt, data: UInt)(implicit conf: TileLinkConfiguration): Acquire = { + def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt, data: UInt): Acquire = { val acq = apply(a_type, addr, client_xact_id, data) acq.subword_addr := subword_addr acq.atomic_opcode := atomic_opcode acq } - def apply(a: Acquire)(implicit conf: TileLinkConfiguration): Acquire = { + def apply(a: Acquire): Acquire = { val acq = new Acquire acq := a acq } } -class Acquire(implicit val tlconf: TileLinkConfiguration) extends ClientSourcedMessage +class Acquire extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasTileLinkData { - val a_type = UInt(width = tlconf.co.acquireTypeWidth) - val write_mask = Bits(width = tlconf.writeMaskBits) - val subword_addr = Bits(width = tlconf.wordAddrBits) - val atomic_opcode = Bits(width = tlconf.atomicOpBits) + val a_type = UInt(width = params(TLCoherence).acquireTypeWidth) + val write_mask = Bits(width = params(TLWriteMaskBits)) + val subword_addr = Bits(width = params(TLWordAddrBits)) + val atomic_opcode = Bits(width = params(TLAtomicOpBits)) } object Probe { - def apply(p_type: UInt, addr: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { + def apply(p_type: UInt, addr: UInt, master_xact_id: UInt) = { val prb = new Probe prb.p_type := p_type prb.addr := addr @@ -85,22 +87,22 @@ object Probe } } -class Probe(implicit val tlconf: TileLinkConfiguration) extends MasterSourcedMessage +class Probe extends MasterSourcedMessage with HasPhysicalAddress with HasMasterTransactionId { - val p_type = UInt(width = tlconf.co.probeTypeWidth) + val p_type = UInt(width = params(TLCoherence).probeTypeWidth) } object Release { - def apply(r_type: UInt, addr: UInt, data: UInt)(implicit conf: TileLinkConfiguration) = { + def apply(r_type: UInt, addr: UInt, data: UInt): Release = { val rel = new Release rel.r_type := r_type rel.addr := addr rel.data := data rel } - def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { + def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt): Release = { val rel = new Release rel.r_type := r_type rel.addr := addr @@ -109,24 +111,24 @@ object Release rel.data := UInt(0) rel } - def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt)(implicit conf: TileLinkConfiguration): Release = { + def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt): Release = { val rel = apply(r_type, addr, client_xact_id, master_xact_id) rel.data := data rel } } -class Release(implicit val tlconf: TileLinkConfiguration) extends ClientSourcedMessage +class Release extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasMasterTransactionId with HasTileLinkData { - val r_type = UInt(width = tlconf.co.releaseTypeWidth) + val r_type = UInt(width = params(TLCoherence).releaseTypeWidth) } object Grant { - def apply(g_type: UInt, client_xact_id: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { + def apply(g_type: UInt, client_xact_id: UInt, master_xact_id: UInt): Grant = { val gnt = new Grant gnt.g_type := g_type gnt.client_xact_id := client_xact_id @@ -134,40 +136,35 @@ object Grant gnt.data := UInt(0) gnt } - def apply(g_type: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt)(implicit conf: TileLinkConfiguration): Grant = { + def apply(g_type: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt): Grant = { val gnt = apply(g_type, client_xact_id, master_xact_id) gnt.data := data gnt } } -class Grant(implicit val tlconf: TileLinkConfiguration) extends MasterSourcedMessage +class Grant extends MasterSourcedMessage with HasTileLinkData with HasClientTransactionId with HasMasterTransactionId { - val g_type = UInt(width = tlconf.co.grantTypeWidth) + val g_type = UInt(width = params(TLCoherence).grantTypeWidth) } -class Finish(implicit val tlconf: TileLinkConfiguration) extends ClientSourcedMessage with HasMasterTransactionId +class Finish extends ClientSourcedMessage with HasMasterTransactionId -class UncachedTileLinkIO(implicit conf: TileLinkConfiguration) extends Bundle { - implicit val ln = conf.ln +class UncachedTileLinkIO extends Bundle { val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) - override def clone = { new UncachedTileLinkIO().asInstanceOf[this.type] } } -class TileLinkIO(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIO()(conf) { +class TileLinkIO extends UncachedTileLinkIO { val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip val release = new DecoupledIO(new LogicalNetworkIO(new Release)) - override def clone = { new TileLinkIO().asInstanceOf[this.type] } } -abstract class TileLinkArbiterLike(val arbN: Int)(implicit conf: TileLinkConfiguration) extends Module { - implicit val (ln, co) = (conf.ln, conf.co) - +abstract class TileLinkArbiterLike(val arbN: Int) extends Module { type MasterSourcedWithId = MasterSourcedMessage with HasClientTransactionId type ClientSourcedWithId = ClientSourcedMessage with HasClientTransactionId @@ -178,7 +175,7 @@ abstract class TileLinkArbiterLike(val arbN: Int)(implicit conf: TileLinkConfigu def hookupClientSource[M <: ClientSourcedWithId] (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], out: DecoupledIO[LogicalNetworkIO[M]]) { - def hasData(m: LogicalNetworkIO[M]) = co.messageHasData(m.payload) + def hasData(m: LogicalNetworkIO[M]) = params(TLCoherence).messageHasData(m.payload) val arb = Module(new RRArbiter(out.bits.clone, arbN)) out <> arb.io.out ins.zipWithIndex.zip(arb.io.in).map{ case ((req,id), arb) => { @@ -205,7 +202,8 @@ abstract class TileLinkArbiterLike(val arbN: Int)(implicit conf: TileLinkConfigu } } -abstract class UncachedTileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkArbiterLike(n)(conf) { +abstract class UncachedTileLinkIOArbiter(n: Int) + extends TileLinkArbiterLike(n) { val io = new Bundle { val in = Vec.fill(n){new UncachedTileLinkIO}.flip val out = new UncachedTileLinkIO @@ -219,7 +217,7 @@ abstract class UncachedTileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfigur finish_arb.io.in zip io.in map { case (arb, req) => arb <> req.finish } } -abstract class TileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkArbiterLike(n)(conf) { +abstract class TileLinkIOArbiter(n: Int) extends TileLinkArbiterLike(n) { val io = new Bundle { val in = Vec.fill(n){new TileLinkIO}.flip val out = new TileLinkIO @@ -267,9 +265,9 @@ abstract trait UsesNewId { in.client_xact_id } -class UncachedTileLinkIOArbiterThatAppendsArbiterId(val n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) with AppendsArbiterId -class UncachedTileLinkIOArbiterThatPassesId(val n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) with PassesId -class UncachedTileLinkIOArbiterThatUsesNewId(val n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) with UsesNewId -class TileLinkIOArbiterThatAppendsArbiterId(val n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkIOArbiter(n)(conf) with AppendsArbiterId -class TileLinkIOArbiterThatPassesId(val n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkIOArbiter(n)(conf) with PassesId -class TileLinkIOArbiterThatUsesNewId(val n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkIOArbiter(n)(conf) with UsesNewId +class UncachedTileLinkIOArbiterThatAppendsArbiterId(val n: Int) extends UncachedTileLinkIOArbiter(n) with AppendsArbiterId +class UncachedTileLinkIOArbiterThatPassesId(val n: Int) extends UncachedTileLinkIOArbiter(n) with PassesId +class UncachedTileLinkIOArbiterThatUsesNewId(val n: Int) extends UncachedTileLinkIOArbiter(n) with UsesNewId +class TileLinkIOArbiterThatAppendsArbiterId(val n: Int) extends TileLinkIOArbiter(n) with AppendsArbiterId +class TileLinkIOArbiterThatPassesId(val n: Int) extends TileLinkIOArbiter(n) with PassesId +class TileLinkIOArbiterThatUsesNewId(val n: Int) extends TileLinkIOArbiter(n) with UsesNewId diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 09bb0714..013f84c2 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -1,33 +1,27 @@ package uncore import Chisel._ -trait CoherenceAgentConfiguration { - def tl: TileLinkConfiguration - def nReleaseTransactions: Int - def nAcquireTransactions: Int -} +case object NReleaseTransactors extends Field[Int] +case object NAcquireTransactors extends Field[Int] +case object NTransactors extends Field[Int] +case object NClients extends Field[Int] -abstract class CoherenceAgent(implicit conf: CoherenceAgentConfiguration) extends Module { +abstract class CoherenceAgent extends Module { + val co = params(TLCoherence) val io = new Bundle { - val inner = (new TileLinkIO()(conf.tl)).flip - val outer = new UncachedTileLinkIO()(conf.tl) - val incoherent = Vec.fill(conf.tl.ln.nClients){Bool()}.asInput + val inner = (new TileLinkIO).flip + val outer = new UncachedTileLinkIO + val incoherent = Vec.fill(params(NClients)){Bool()}.asInput } } -case class L2CoherenceAgentConfiguration( - val tl: TileLinkConfiguration, - val nReleaseTransactions: Int, - val nAcquireTransactions: Int) extends CoherenceAgentConfiguration - -class L2CoherenceAgent(bankId: Int)(implicit conf: CoherenceAgentConfiguration) - extends CoherenceAgent { - implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) +class L2CoherenceAgent(bankId: Int) extends CoherenceAgent { // Create SHRs for outstanding transactions - val nTrackers = conf.nReleaseTransactions + conf.nAcquireTransactions - val trackerList = (0 until conf.nReleaseTransactions).map(id => Module(new VoluntaryReleaseTracker(id, bankId))) ++ - (conf.nReleaseTransactions until nTrackers).map(id => Module(new AcquireTracker(id, bankId))) + val trackerList = (0 until params(NReleaseTransactors)).map(id => + Module(new VoluntaryReleaseTracker(id, bankId))) ++ + (params(NReleaseTransactors) until params(NTransactors)).map(id => + Module(new AcquireTracker(id, bankId))) // Propagate incoherence flags trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) @@ -85,12 +79,12 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: CoherenceAgentConfiguration) } -abstract class XactTracker()(implicit conf: CoherenceAgentConfiguration) extends Module { - implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) +abstract class XactTracker extends Module { + val (co, nClients) = (params(TLCoherence),params(NClients)) val io = new Bundle { val inner = (new TileLinkIO).flip val outer = new UncachedTileLinkIO - val tile_incoherent = Bits(INPUT, ln.nClients) + val tile_incoherent = Bits(INPUT, params(NClients)) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) } @@ -100,15 +94,13 @@ abstract class XactTracker()(implicit conf: CoherenceAgentConfiguration) extends val c_gnt = io.inner.grant.bits val c_ack = io.inner.finish.bits val m_gnt = io.outer.grant.bits - } -class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentConfiguration) - extends XactTracker()(conf) { +class VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends XactTracker { val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) val xact = Reg{ new Release } - val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) val incoming_rel = io.inner.release.bits io.has_acquire_conflict := Bool(false) @@ -153,17 +145,16 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: Cohere } } -class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentConfiguration) - extends XactTracker()(conf) { +class AcquireTracker(trackerId: Int, bankId: Int) extends XactTracker { val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) val state = Reg(init=s_idle) val xact = Reg{ new Acquire } - val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) //TODO: Will need id reg for merged release xacts - val init_sharer_cnt = Reg(init=UInt(0, width = log2Up(ln.nClients))) - val release_count = if (ln.nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(ln.nClients))) - val probe_flags = Reg(init=Bits(0, width = ln.nClients)) + val init_sharer_cnt = Reg(init=UInt(0, width = log2Up(nClients))) + val release_count = if (nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(nClients))) + val probe_flags = Reg(init=Bits(0, width = nClients)) val curr_p_id = PriorityEncoder(probe_flags) val pending_outer_write = co.messageHasData(xact) @@ -174,12 +165,12 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC xact.addr, UInt(trackerId), c_rel.payload.data) val outer_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) - val probe_initial_flags = Bits(width = ln.nClients) + val probe_initial_flags = Bits(width = nClients) probe_initial_flags := Bits(0) - if (ln.nClients > 1) { + if (nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence val probe_self = Bool(true) //co.needsSelfProbe(io.inner.acquire.bits.payload) - val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(ln.nClients)-1,0))) + val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } @@ -219,9 +210,9 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src - init_sharer_cnt := UInt(ln.nClients) // TODO: Broadcast only + init_sharer_cnt := UInt(nClients) // TODO: Broadcast only probe_flags := probe_initial_flags - if(ln.nClients > 1) { + if(nClients > 1) { release_count := PopCount(probe_initial_flags) state := Mux(probe_initial_flags.orR, s_probe, Mux(needs_outer_write, s_mem_write, @@ -244,7 +235,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC io.outer.acquire.bits.payload := outer_write_rel when(io.outer.acquire.ready) { io.inner.release.ready := Bool(true) - if(ln.nClients > 1) release_count := release_count - UInt(1) + if(nClients > 1) release_count := release_count - UInt(1) when(release_count === UInt(1)) { state := Mux(pending_outer_write, s_mem_write, Mux(pending_outer_read, s_mem_read, s_make_grant)) @@ -252,7 +243,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC } } .otherwise { io.inner.release.ready := Bool(true) - if(ln.nClients > 1) release_count := release_count - UInt(1) + if(nClients > 1) release_count := release_count - UInt(1) when(release_count === UInt(1)) { state := Mux(pending_outer_write, s_mem_write, Mux(pending_outer_read, s_mem_read, s_make_grant)) From 9ab3a4262cedb689d616c6427f20f88d2e40521d Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 11 Aug 2014 18:35:49 -0700 Subject: [PATCH 233/374] Cache utility traits. Completely compiles, asm tests hang. --- uncore/src/main/scala/cache.scala | 113 +++++++++++++++----------- uncore/src/main/scala/memserdes.scala | 2 +- uncore/src/main/scala/uncore.scala | 17 ++-- 3 files changed, 80 insertions(+), 52 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 247110a5..21cf582e 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -3,17 +3,35 @@ import Chisel._ case object NSets extends Field[Int] case object NWays extends Field[Int] -case object IsDM extends Field[Boolean] -case object TagBits extends Field[Int] -case object IdxBits extends Field[Int] -case object OffBits extends Field[Int] -case object UntagBits extends Field[Int] -case object WayBits extends Field[Int] +case object BlockOffBits extends Field[Int] case object RowBits extends Field[Int] case object WordBits extends Field[Int] -case object RefillCycles extends Field[Int] case object Replacer extends Field[() => ReplacementPolicy] +abstract trait CacheParameters extends UsesParameters { + val paddrBits = params(PAddrBits) + val vaddrBits = params(VAddrBits) + val pgIdxBits = params(PgIdxBits) + val nSets = params(NSets) + val nWays = params(NWays) + val blockOffBits = params(BlockOffBits) + val wordBits = params(WordBits) + val rowBits = params(RowBits) + val wordOffBits = log2Up(wordBits) + val idxBits = log2Up(nSets) + val untagBits = blockOffBits + idxBits + val tagBits = paddrBits - untagBits + val wayBits = log2Up(nWays) + val isDM = nWays == 1 + val rowWords = rowBits/wordBits + val rowBytes = rowBits*8 + val rowOffBits = log2Up(rowBytes) + val refillCycles = params(TLDataBits)/rowBits +} + +abstract class CacheBundle extends Bundle with CacheParameters +abstract class CacheModule extends Module with CacheParameters + abstract class ReplacementPolicy { def way: UInt def miss: Unit @@ -30,43 +48,43 @@ class RandomReplacement(ways: Int) extends ReplacementPolicy { def hit = {} } -abstract class Metadata extends Bundle { - val tag = Bits(width = params(TagBits)) +abstract class Metadata extends CacheBundle { + val tag = Bits(width = tagBits) val coh: CoherenceMetadata } -class MetaReadReq extends Bundle { - val idx = Bits(width = params(IdxBits)) +class MetaReadReq extends CacheBundle { + val idx = Bits(width = idxBits) } class MetaWriteReq[T <: Metadata](gen: T) extends MetaReadReq { - val way_en = Bits(width = params(WayBits)) + val way_en = Bits(width = nWays) val data = gen.clone override def clone = new MetaWriteReq(gen).asInstanceOf[this.type] } -class MetadataArray[T <: Metadata](makeRstVal: () => T) extends Module { +class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { val rstVal = makeRstVal() val io = new Bundle { val read = Decoupled(new MetaReadReq).flip val write = Decoupled(new MetaWriteReq(rstVal.clone)).flip - val resp = Vec.fill(params(NWays)){rstVal.clone.asOutput} + val resp = Vec.fill(nWays){rstVal.clone.asOutput} } val metabits = rstVal.getWidth - val rst_cnt = Reg(init=UInt(0, log2Up(params(NSets)+1))) - val rst = rst_cnt < UInt(params(NSets)) + val rst_cnt = Reg(init=UInt(0, log2Up(nSets+1))) + val rst = rst_cnt < UInt(nSets) val waddr = Mux(rst, rst_cnt, io.write.bits.idx) val wdata = Mux(rst, rstVal, io.write.bits.data).toBits val wmask = Mux(rst, SInt(-1), io.write.bits.way_en) when (rst) { rst_cnt := rst_cnt+UInt(1) } - val tag_arr = Mem(UInt(width = metabits*params(NWays)), params(NSets), seqRead = true) + val tag_arr = Mem(UInt(width = metabits*nWays), nSets, seqRead = true) when (rst || io.write.valid) { - tag_arr.write(waddr, Fill(params(NWays), wdata), FillInterleaved(metabits, wmask)) + tag_arr.write(waddr, Fill(nWays, wdata), FillInterleaved(metabits, wmask)) } val tags = tag_arr(RegEnable(io.read.bits.idx, io.read.valid)) - for (w <- 0 until params(NWays)) { + for (w <- 0 until nWays) { val m = tags(metabits*(w+1)-1, metabits*w) io.resp(w) := rstVal.clone.fromBits(m) } @@ -75,14 +93,20 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends Module { io.write.ready := !rst } -trait HasL2Id extends Bundle { - val id = UInt(width = log2Up(params(NTransactors))) +abstract trait L2HellaCacheParameters extends CacheParameters + with CoherenceAgentParameters + +abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters +abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters + +trait HasL2Id extends Bundle with CoherenceAgentParameters { + val id = UInt(width = log2Up(nTransactors)) } -trait HasL2InternalRequestState extends Bundle { +trait HasL2InternalRequestState extends L2HellaCacheBundle { val tag_match = Bool() val old_meta = new L2Metadata - val way_en = Bits(width = params(NWays)) + val way_en = Bits(width = nWays) } object L2Metadata { @@ -93,23 +117,22 @@ object L2Metadata { meta } } -class L2Metadata extends Metadata { - val coh = params(TLCoherence).masterMetadataOnFlush.clone +class L2Metadata extends Metadata with L2HellaCacheParameters { + val coh = co.masterMetadataOnFlush.clone } class L2MetaReadReq extends MetaReadReq with HasL2Id { - val tag = Bits(width = params(TagBits)) + val tag = Bits(width = tagBits) } class L2MetaWriteReq extends MetaWriteReq[L2Metadata](new L2Metadata) with HasL2Id -class L2MetaResp extends Bundle +class L2MetaResp extends L2HellaCacheBundle with HasL2Id with HasL2InternalRequestState -class L2MetadataArray extends Module { - val (co, ways) = (params(TLCoherence), params(NWays)) +class L2MetadataArray extends L2HellaCacheModule { val io = new Bundle { val read = Decoupled(new L2MetaReadReq).flip val write = Decoupled(new L2MetaWriteReq).flip @@ -123,7 +146,7 @@ class L2MetadataArray extends Module { val s1_clk_en = Reg(next = io.read.fire()) val s1_tag = RegEnable(io.read.bits.tag, io.read.valid) val s1_id = RegEnable(io.read.bits.id, io.read.valid) - def wayMap[T <: Data](f: Int => T) = Vec((0 until ways).map(f)) + def wayMap[T <: Data](f: Int => T) = Vec((0 until nWays).map(f)) val s1_tag_eq_way = wayMap((w: Int) => meta.io.resp(w).tag === s1_tag) val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && co.isValid(meta.io.resp(w).coh)).toBits val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) @@ -146,8 +169,8 @@ class L2MetadataArray extends Module { io.resp.bits.way_en := Mux(s2_tag_match, s2_tag_match_way, s2_replaced_way_en) } -class L2DataReadReq extends Bundle with HasL2Id { - val way_en = Bits(width = params(NWays)) +class L2DataReadReq extends L2HellaCacheBundle with HasL2Id { + val way_en = Bits(width = nWays) val addr = Bits(width = params(TLAddrBits)) } @@ -160,7 +183,7 @@ class L2DataResp extends Bundle with HasL2Id { val data = Bits(width = params(TLDataBits)) } -class L2DataArray extends Module { +class L2DataArray extends L2HellaCacheModule { val io = new Bundle { val read = Decoupled(new L2DataReadReq).flip val write = Decoupled(new L2DataWriteReq).flip @@ -169,9 +192,9 @@ class L2DataArray extends Module { val waddr = io.write.bits.addr val raddr = io.read.bits.addr - val wmask = FillInterleaved(params(WordBits), io.write.bits.wmask) - val resp = (0 until params(NWays)).map { w => - val array = Mem(Bits(width=params(RowBits)), params(NSets)*params(RefillCycles), seqRead = true) + val wmask = FillInterleaved(wordBits, io.write.bits.wmask) + val resp = (0 until nWays).map { w => + val array = Mem(Bits(width=params(RowBits)), nSets*refillCycles, seqRead = true) when (io.write.bits.way_en(w) && io.write.valid) { array.write(waddr, io.write.bits.data, wmask) } @@ -185,11 +208,11 @@ class L2DataArray extends Module { io.write.ready := Bool(true) } -class L2HellaCache(bankId: Int) extends CoherenceAgent { +class L2HellaCache(bankId: Int) extends CoherenceAgent with L2HellaCacheParameters { - require(isPow2(params(NSets))) - require(isPow2(params(NWays))) - require(params(RefillCycles) == 1) + require(isPow2(nSets)) + require(isPow2(nWays)) + require(refillCycles == 1) val tshrfile = Module(new TSHRFile(bankId)) val meta = Module(new L2MetadataArray) @@ -207,8 +230,7 @@ class L2HellaCache(bankId: Int) extends CoherenceAgent { } -class TSHRFile(bankId: Int) extends Module { - val (co, nClients) = (params(TLCoherence), params(NClients)) +class TSHRFile(bankId: Int) extends L2HellaCacheModule { val io = new Bundle { val inner = (new TileLinkIO).flip val outer = new UncachedTileLinkIO @@ -234,9 +256,9 @@ class TSHRFile(bankId: Int) extends Module { } // Create TSHRs for outstanding transactions - val trackerList = (0 until params(NReleaseTransactors)).map { id => + val trackerList = (0 until nReleaseTransactors).map { id => Module(new L2VoluntaryReleaseTracker(id, bankId)) - } ++ (params(NReleaseTransactors) until params(NTransactors)).map { id => + } ++ (nReleaseTransactors until nTransactors).map { id => Module(new L2AcquireTracker(id, bankId)) } @@ -301,8 +323,7 @@ class TSHRFile(bankId: Int) extends Module { } -abstract class L2XactTracker extends Module { - val (co, nClients) = (params(TLCoherence),params(NClients)) +abstract class L2XactTracker extends L2HellaCacheModule { val io = new Bundle { val inner = (new TileLinkIO).flip val outer = new UncachedTileLinkIO diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index eba971fd..02d74385 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -209,7 +209,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { val co = params(TLCoherence) val tbits = params(MIFTagBits) val dbits = params(MIFDataBits) - val dbeats = params(MIFDataBits) + val dbeats = params(MIFDataBeats) require(params(TLDataBits) == dbits*dbeats) //require(params(TLClientXactIdBits) <= params(MIFTagBits)) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 013f84c2..580f0477 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -3,24 +3,31 @@ import Chisel._ case object NReleaseTransactors extends Field[Int] case object NAcquireTransactors extends Field[Int] -case object NTransactors extends Field[Int] case object NClients extends Field[Int] -abstract class CoherenceAgent extends Module { +abstract trait CoherenceAgentParameters extends UsesParameters { val co = params(TLCoherence) + val nReleaseTransactors = params(NReleaseTransactors) + val nAcquireTransactors = params(NAcquireTransactors) + val nTransactors = nReleaseTransactors + nAcquireTransactors + val nClients = params(NClients) +} + +abstract class CoherenceAgent extends Module + with CoherenceAgentParameters { val io = new Bundle { val inner = (new TileLinkIO).flip val outer = new UncachedTileLinkIO - val incoherent = Vec.fill(params(NClients)){Bool()}.asInput + val incoherent = Vec.fill(nClients){Bool()}.asInput } } class L2CoherenceAgent(bankId: Int) extends CoherenceAgent { // Create SHRs for outstanding transactions - val trackerList = (0 until params(NReleaseTransactors)).map(id => + val trackerList = (0 until nReleaseTransactors).map(id => Module(new VoluntaryReleaseTracker(id, bankId))) ++ - (params(NReleaseTransactors) until params(NTransactors)).map(id => + (nReleaseTransactors until nTransactors).map(id => Module(new AcquireTracker(id, bankId))) // Propagate incoherence flags From e26f8a6f6ae8c1da134939a9c4177752f34a5a3c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 12 Aug 2014 14:55:44 -0700 Subject: [PATCH 234/374] Fix errors in derived cache params --- uncore/src/main/scala/cache.scala | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 21cf582e..d52e692a 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -13,18 +13,19 @@ abstract trait CacheParameters extends UsesParameters { val vaddrBits = params(VAddrBits) val pgIdxBits = params(PgIdxBits) val nSets = params(NSets) - val nWays = params(NWays) val blockOffBits = params(BlockOffBits) - val wordBits = params(WordBits) - val rowBits = params(RowBits) - val wordOffBits = log2Up(wordBits) val idxBits = log2Up(nSets) val untagBits = blockOffBits + idxBits val tagBits = paddrBits - untagBits + val nWays = params(NWays) val wayBits = log2Up(nWays) val isDM = nWays == 1 + val wordBits = params(WordBits) + val wordBytes = wordBits/8 + val wordOffBits = log2Up(wordBytes) + val rowBits = params(RowBits) val rowWords = rowBits/wordBits - val rowBytes = rowBits*8 + val rowBytes = rowBits/8 val rowOffBits = log2Up(rowBytes) val refillCycles = params(TLDataBits)/rowBits } From e384b33cc36c8ecbb0fe5bf5f87ee7f4e84b29af Mon Sep 17 00:00:00 2001 From: Scott Beamer Date: Tue, 19 Aug 2014 15:50:20 -0700 Subject: [PATCH 235/374] don't generate a write mask for BigMem if it isn't used not needed for llc data --- uncore/src/main/scala/llc.scala | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index 3a21cdca..09ad87bb 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -1,7 +1,7 @@ package uncore import Chisel._ -class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UInt])(gen: => T) extends Module +class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UInt], noMask: Boolean = false)(gen: => T) extends Module { class Inputs extends Bundle { val addr = UInt(INPUT, log2Up(n)) @@ -44,7 +44,12 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn wmask0 = wmask0 & FillInterleaved(gen.getWidth, UIntToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.getWidth*(j+1))-1, leaf.data.getWidth*j)) when (in.valid) { - when (in.bits.rw) { mem.write(idx, wdata0, wmask0) } + when (in.bits.rw) { + if (noMask) + mem.write(idx, wdata0) + else + mem.write(idx, wdata0, wmask0) + } .otherwise { if (postLatency > 0) ridx := idx } } @@ -245,7 +250,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[ val mem_resp_way = UInt(INPUT, log2Up(ways)) } - val data = Module(new BigMem(sets*ways*refill_cycles, 1, latency-1, leaf)(Bits(width = conf.dataBits))) + val data = Module(new BigMem(sets*ways*refill_cycles, 1, latency-1, leaf, true)(Bits(width = conf.dataBits))) class QEntry extends MemResp { val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] @@ -268,7 +273,6 @@ class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[ data.io.in.bits.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUInt data.io.in.bits.rw := io.req.bits.rw data.io.in.bits.wdata := io.req_data.bits.data - data.io.in.bits.wmask := SInt(-1, io.req_data.bits.data.getWidth) when (valid) { data.io.in.valid := Mux(req.rw, io.req_data.valid, qReady) data.io.in.bits.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUInt From dc5643b12f2e657aa95712af92dceb4effdc3eff Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sat, 23 Aug 2014 01:19:36 -0700 Subject: [PATCH 236/374] Final parameter refactor. --- uncore/src/main/scala/cache.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index d52e692a..c35aeedf 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -1,6 +1,7 @@ package uncore import Chisel._ +case object CacheName extends Field[String] case object NSets extends Field[Int] case object NWays extends Field[Int] case object BlockOffBits extends Field[Int] From 17b2359c9ae70882cb4e309afe7bdc47bc11f33f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 24 Aug 2014 19:27:58 -0700 Subject: [PATCH 237/374] htif parameters trait --- uncore/src/main/scala/htif.scala | 45 +++++++++++++++++--------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 153bad3a..bfe4ef10 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -9,7 +9,18 @@ case object HTIFNSCR extends Field[Int] case object HTIFOffsetBits extends Field[Int] case object HTIFNCores extends Field[Int] -class HostIO(val w: Int) extends Bundle +abstract trait HTIFParameters extends UsesParameters { + val dataBits = params(TLDataBits) + val co = params(TLCoherence) + val w = params(HTIFWidth) + val nSCR = params(HTIFNSCR) + val offsetBits = params(HTIFOffsetBits) + val nCores = params(HTIFNCores) +} + +abstract class HTIFBundle extends Bundle with HTIFParameters + +class HostIO extends HTIFBundle { val clk = Bool(OUTPUT) val clk_edge = Bool(OUTPUT) @@ -25,42 +36,34 @@ class PCRReq extends Bundle val data = Bits(width = 64) } -class HTIFIO extends Bundle -{ +class HTIFIO extends HTIFBundle { val reset = Bool(INPUT) - val id = UInt(INPUT, log2Up(params(HTIFNCores))) + val id = UInt(INPUT, log2Up(nCores)) val pcr_req = Decoupled(new PCRReq).flip val pcr_rep = Decoupled(Bits(width = 64)) - val ipi_req = Decoupled(Bits(width = log2Up(params(HTIFNCores)))) + val ipi_req = Decoupled(Bits(width = log2Up(nCores))) val ipi_rep = Decoupled(Bool()).flip val debug_stats_pcr = Bool(OUTPUT) // wired directly to stats register // expected to be used to quickly indicate to testbench to do logging b/c in 'interesting' work } -class SCRIO extends Bundle -{ - val rdata = Vec.fill(params(HTIFNSCR)){Bits(INPUT, 64)} +class SCRIO extends HTIFBundle { + val rdata = Vec.fill(nSCR){Bits(INPUT, 64)} val wen = Bool(OUTPUT) - val waddr = UInt(OUTPUT, log2Up(params(HTIFNSCR))) + val waddr = UInt(OUTPUT, log2Up(nSCR)) val wdata = Bits(OUTPUT, 64) } -class HTIF(pcr_RESET: Int) extends Module -{ - val dataBits = params(TLDataBits) - val co = params(TLCoherence) - val w = params(HTIFWidth) - val nSCR = params(HTIFNSCR) - val offsetBits = params(HTIFOffsetBits) - val nCores = params(HTIFNCores) - val io = new Bundle { - val host = new HostIO(w) - val cpu = Vec.fill(nCores){new HTIFIO().flip} +class HTIFModuleIO extends HTIFBundle { + val host = new HostIO + val cpu = Vec.fill(nCores){new HTIFIO}.flip val mem = new TileLinkIO val scr = new SCRIO - } +} +class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { + val io = new HTIFModuleIO io.host.debug_stats_pcr := io.cpu.map(_.debug_stats_pcr).reduce(_||_) // system is 'interesting' if any tile is 'interesting' From 712f3a754d73c3fa8149e04596d9b93dfd891658 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 2 Sep 2014 12:34:42 -0700 Subject: [PATCH 238/374] merge in master --- uncore/src/main/scala/htif.scala | 2 +- uncore/src/main/scala/llc.scala | 31 ++++++++++++++++++------------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index bfe4ef10..8d1b2618 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -104,7 +104,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { val cmd_readmem :: cmd_writemem :: cmd_readcr :: cmd_writecr :: cmd_ack :: cmd_nack :: Nil = Enum(UInt(), 6) - val pcr_addr = addr(io.cpu(0).pcr_req.bits.addr.width-1, 0) + val pcr_addr = addr(io.cpu(0).pcr_req.bits.addr.getWidth-1, 0) val pcr_coreid = addr(log2Up(nCores)-1+20+1,20) val pcr_wdata = packet_ram(0) diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index 5bc89e38..f34ffac4 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -1,7 +1,7 @@ package uncore import Chisel._ -class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UInt])(gen: => T) extends Module +class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UInt], noMask: Boolean = false)(gen: => T) extends Module { class Inputs extends Bundle { val addr = UInt(INPUT, log2Up(n)) @@ -15,8 +15,8 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn val rdata = gen.asOutput } val data = gen - val colMux = if (2*data.width <= leaf.data.width && n > leaf.n) 1 << math.floor(math.log(leaf.data.width/data.width)/math.log(2)).toInt else 1 - val nWide = if (data.width > leaf.data.width) 1+(data.width-1)/leaf.data.width else 1 + val colMux = if (2*data.getWidth <= leaf.data.getWidth && n > leaf.n) 1 << math.floor(math.log(leaf.data.getWidth/data.getWidth)/math.log(2)).toInt else 1 + val nWide = if (data.getWidth > leaf.data.getWidth) 1+(data.getWidth-1)/leaf.data.getWidth else 1 val nDeep = if (n > colMux*leaf.n) 1+(n-1)/(colMux*leaf.n) else 1 if (nDeep > 1 || colMux > 1) require(isPow2(n) && isPow2(leaf.n)) @@ -39,12 +39,17 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn var dout: Bits = null val ridx = if (postLatency > 0) Reg(Bits()) else null - var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) + var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.getWidth*(j+1))-1, leaf.data.getWidth*j)) if (colMux > 1) - wmask0 = wmask0 & FillInterleaved(gen.width, UIntToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) - val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) + wmask0 = wmask0 & FillInterleaved(gen.getWidth, UIntToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) + val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.getWidth*(j+1))-1, leaf.data.getWidth*j)) when (in.valid) { - when (in.bits.rw) { mem.write(idx, wdata0, wmask0) } + when (in.bits.rw) { + if (noMask) + mem.write(idx, wdata0) + else + mem.write(idx, wdata0, wmask0) + } .otherwise { if (postLatency > 0) ridx := idx } } @@ -61,7 +66,7 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn var colMuxOut = rdataWide if (colMux > 1) { - val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.width*(k+1)-1, gen.width*k))) + val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.getWidth*(k+1)-1, gen.getWidth*k))) colMuxOut = colMuxIn(r.bits(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) } @@ -245,7 +250,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[ val mem_resp_way = UInt(INPUT, log2Up(ways)) } - val data = Module(new BigMem(sets*ways*refill_cycles, 1, latency-1, leaf)(Bits(width = params(MIFDataBits)))) + val data = Module(new BigMem(sets*ways*refill_cycles, 1, latency-1, leaf, true)(Bits(width = params(MIFDataBits)))) class QEntry extends MemResp { val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] @@ -268,7 +273,6 @@ class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[ data.io.in.bits.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUInt data.io.in.bits.rw := io.req.bits.rw data.io.in.bits.wdata := io.req_data.bits.data - data.io.in.bits.wmask := SInt(-1, io.req_data.bits.data.width) when (valid) { data.io.in.valid := Mux(req.rw, io.req_data.valid, qReady) data.io.in.bits.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUInt @@ -385,7 +389,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, ta for (i <- 0 until ways) s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) } - val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.width-1, s2.addr.width-tagWidth) === t(tagWidth-1, 0)) + val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.getWidth-1, s2.addr.getWidth-tagWidth) === t(tagWidth-1, 0)) val s2_hit_way = OHToUInt(s2_hits) val s2_hit = s2_hits.reduceLeft(_||_) val s2_hit_dirty = s2_tags(s2_hit_way)(tagWidth+1) @@ -395,7 +399,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, ta val tag_we = initialize || setDirty || mshr.io.tag.fire() val tag_waddr = Mux(initialize, initCount, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)) - val tag_wdata = Cat(setDirty, !initialize, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)) + val tag_wdata = Cat(setDirty, !initialize, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.getWidth-1, mshr.io.tag.bits.addr.getWidth-tagWidth)) val tag_wmask = Mux(initialize, SInt(-1, ways), UIntToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way))) tags.io.in.valid := io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy || tag_we tags.io.in.bits.addr := Mux(tag_we, tag_waddr, Mux(replay_s2, s2.addr, io.cpu.req_cmd.bits.addr)(log2Up(sets)-1,0)) @@ -438,10 +442,11 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, ta replay_s2_rdy := s3_rdy && !tag_we io.cpu.resp <> data.io.resp - io.cpu.req_cmd.ready := !s1_valid && !s2_valid && !replay_s2 && !tag_we io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready io.mem.req_cmd <> memCmdArb.io.out io.mem.req_data <> writeback.io.mem.req_data + io.cpu.req_cmd.ready := !(s1_valid || s2_valid || replay_s2 || tag_we || + io.cpu.req_cmd.bits.rw && io.cpu.req_data.ready) } class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module From bfb662968d2e34ad589988866a4638900c5ad39e Mon Sep 17 00:00:00 2001 From: Scott Beamer Date: Tue, 2 Sep 2014 14:33:58 -0700 Subject: [PATCH 239/374] fixes sbt error during first run --- uncore/chisel-dependent.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/chisel-dependent.sbt b/uncore/chisel-dependent.sbt index fa8d5490..eeb52c04 100644 --- a/uncore/chisel-dependent.sbt +++ b/uncore/chisel-dependent.sbt @@ -1,6 +1,6 @@ // Provide a managed dependency on chisel if -DchiselVersion="" is // supplied on the command line. -val chiselVersion = System.getProperty("chiselVersion", "None") +lazy val chiselVersion = System.getProperty("chiselVersion", "None") libraryDependencies ++= ( if (chiselVersion != "None" ) ("edu.berkeley.cs" %% "chisel" % chiselVersion) :: Nil; else Nil) From f8821b4cc9a9878c76b6ab849f31914163da4e04 Mon Sep 17 00:00:00 2001 From: Scott Beamer Date: Tue, 2 Sep 2014 15:16:03 -0700 Subject: [PATCH 240/374] better fix with explanation of sbt issue --- uncore/chisel-dependent.sbt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/uncore/chisel-dependent.sbt b/uncore/chisel-dependent.sbt index eeb52c04..88eb615c 100644 --- a/uncore/chisel-dependent.sbt +++ b/uncore/chisel-dependent.sbt @@ -1,6 +1,8 @@ // Provide a managed dependency on chisel if -DchiselVersion="" is // supplied on the command line. -lazy val chiselVersion = System.getProperty("chiselVersion", "None") +val chiselVersion_u = System.getProperty("chiselVersion", "None") -libraryDependencies ++= ( if (chiselVersion != "None" ) ("edu.berkeley.cs" %% "chisel" % chiselVersion) :: Nil; else Nil) +// _u a temporary fix until sbt 13.6 https://github.com/sbt/sbt/issues/1465 + +libraryDependencies ++= ( if (chiselVersion_u != "None" ) ("edu.berkeley.cs" %% "chisel" % chiselVersion_u) :: Nil; else Nil) From f8d450b4e2156d73cbdb383c4eed19cbbd87ab0c Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Thu, 11 Sep 2014 22:06:03 -0700 Subject: [PATCH 241/374] mark DRAMSideLLC as HasKnownBug --- uncore/src/main/scala/llc.scala | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index f34ffac4..1878a356 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -354,7 +354,19 @@ abstract class DRAMSideLLCLike extends Module { } } -class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, tagLeaf: Mem[UInt], dataLeaf: Mem[UInt]) extends DRAMSideLLCLike +// DRAMSideLLC has a known bug now. DO NOT USE. We are working on a brand new +// L2$. Stay stuned. +// +// Bug description: +// There's a race condition between the writeback unit (the module which +// sends the data out to the backside interface) and the data unit (the module +// which writes the data into the SRAM in the L2$). The "hit status" +// is saved in a register, however, is updated again when there's +// a transaction coming from the core without waiting until the writeback unit +// has sent out all its data to the outer memory system. This is why the +// problem manifests at a higher probability with the slow backup memory port. + +class DRAMSideLLC_HasKnownBug(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, tagLeaf: Mem[UInt], dataLeaf: Mem[UInt]) extends DRAMSideLLCLike { val tagWidth = params(MIFAddrBits) - log2Up(sets) val metaWidth = tagWidth + 2 // valid + dirty @@ -505,7 +517,6 @@ object HellaQueue } class DRAMSideLLCNull(numRequests: Int, refillCycles: Int) extends DRAMSideLLCLike { - val numEntries = numRequests * refillCycles val size = log2Down(numEntries) + 1 From 0b51d70bd288d9146c5c8d99d5916a208c82789b Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Fri, 12 Sep 2014 15:31:38 -0700 Subject: [PATCH 242/374] add LICENSE --- uncore/src/main/scala/cache.scala | 2 ++ uncore/src/main/scala/coherence.scala | 2 ++ uncore/src/main/scala/consts.scala | 2 ++ uncore/src/main/scala/htif.scala | 2 ++ uncore/src/main/scala/llc.scala | 2 ++ uncore/src/main/scala/memserdes.scala | 2 ++ uncore/src/main/scala/network.scala | 2 ++ uncore/src/main/scala/package.scala | 2 ++ uncore/src/main/scala/slowio.scala | 2 ++ uncore/src/main/scala/tilelink.scala | 2 ++ uncore/src/main/scala/uncore.scala | 2 ++ 11 files changed, 22 insertions(+) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index c35aeedf..686f0ac1 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package uncore import Chisel._ diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 45a1970c..5fa0b7a6 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package uncore import Chisel._ diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index 64d6c3a7..a402e7db 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package uncore package constants diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 8d1b2618..edcd66cb 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package uncore import Chisel._ diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index 1878a356..35553702 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package uncore import Chisel._ diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 02d74385..6f42f010 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package uncore import Chisel._ import scala.math._ diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index 72002d45..d3f4a9e8 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package uncore import Chisel._ diff --git a/uncore/src/main/scala/package.scala b/uncore/src/main/scala/package.scala index 501d511d..2c6c4a5f 100644 --- a/uncore/src/main/scala/package.scala +++ b/uncore/src/main/scala/package.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package object uncore extends uncore.constants.MemoryOpConstants { implicit def toOption[A](a: A) = Option(a) diff --git a/uncore/src/main/scala/slowio.scala b/uncore/src/main/scala/slowio.scala index 9ed0abb7..95ca34e6 100644 --- a/uncore/src/main/scala/slowio.scala +++ b/uncore/src/main/scala/slowio.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package uncore import Chisel._ diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index d49e264b..af736c97 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package uncore import Chisel._ diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 580f0477..8f1a7a67 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package uncore import Chisel._ From 49b027db2c8a423b76cb63bf19089f5077823138 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Fri, 12 Sep 2014 15:36:29 -0700 Subject: [PATCH 243/374] forgot to add LICENSE file --- uncore/LICENSE | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 uncore/LICENSE diff --git a/uncore/LICENSE b/uncore/LICENSE new file mode 100644 index 00000000..7cff15e4 --- /dev/null +++ b/uncore/LICENSE @@ -0,0 +1,24 @@ +Copyright (c) 2012-2014, The Regents of the University of California +(Regents). All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. Neither the name of the Regents nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, +SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING +OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS +BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED +HEREUNDER IS PROVIDED "AS IS". REGENTS HAS NO OBLIGATION TO PROVIDE +MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. From f249da1803d958804e5d1e715e324c11b4c1a913 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Wed, 17 Sep 2014 11:25:14 -0700 Subject: [PATCH 244/374] update README --- uncore/README | 6 ------ uncore/README.md | 11 +++++++++++ 2 files changed, 11 insertions(+), 6 deletions(-) delete mode 100644 uncore/README create mode 100644 uncore/README.md diff --git a/uncore/README b/uncore/README deleted file mode 100644 index c920e19a..00000000 --- a/uncore/README +++ /dev/null @@ -1,6 +0,0 @@ -This is the repository for uncore components assosciated with UCB_BAR chip -projects. To uses these modules, include this repo as a git submodule within -the your chip repository and add it as Project in your chip's build.scala. -These components are only dependent on Chisel, i.e. -lazy val uncore = Project("uncore", file("uncore"), settings = buildSettings) dependsOn(chisel) - diff --git a/uncore/README.md b/uncore/README.md new file mode 100644 index 00000000..003f9caa --- /dev/null +++ b/uncore/README.md @@ -0,0 +1,11 @@ +Uncore Library +============== + +This is the repository for uncore components assosciated with Rocket chip +project. To uses these modules, include this repo as a git submodule within +the your chip repository and add it as Project in your chip's build.scala. +These components are only dependent on Chisel, i.e. + + lazy val uncore = Project("uncore", file("uncore"), settings = buildSettings) dependsOn(chisel) + +Documentation about the uncore library will come in the near future. From f7b1e23eadebdc956b68dbd6f5e8eedb04bb6241 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 10 Jul 2014 16:59:48 -0700 Subject: [PATCH 245/374] functional style on MuxBundle --- uncore/src/main/scala/coherence.scala | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 5fa0b7a6..1a3c4573 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -5,11 +5,7 @@ import Chisel._ object MuxBundle { def apply[T <: Data] (default: T, mapping: Seq[(Bool, T)]): T = { - var res = default; - for ((t, v) <- mapping.reverse){ - res = Mux(t, v, res); - } - res + mapping.reverse.foldLeft(default)((b, a) => Mux(a._1, a._2, b)) } } From faed47d131211fa12c393ae5fd864319cfc90ffc Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 10 Jul 2014 17:01:19 -0700 Subject: [PATCH 246/374] use thunk for dir info --- uncore/src/main/scala/coherence.scala | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 1a3c4573..abacdabd 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -39,7 +39,7 @@ object MasterMetadata { } class MasterMetadata(implicit c: CoherencePolicy) extends CoherenceMetadata { val state = UInt(width = c.masterStateWidth) - val sharers = c.dir.clone + val sharers = c.dir() override def clone = new MasterMetadata()(c).asInstanceOf[this.type] } /* @@ -75,10 +75,12 @@ class FullRepresentation(nClients: Int) extends DirectoryRepresentation { def flush(dummy: Int = 0) = { sharers := UInt(0, width = nClients); this } def none(dummy: Int = 0) = sharers === UInt(0) def one(dummy: Int = 0) = PopCount(sharers) === UInt(1) + def count(dummy: Int = 0) = PopCount(sharers) + def next(dummy: Int = 0) = PriorityEncoder(sharers) override def clone = new FullRepresentation(nClients).asInstanceOf[this.type] } -abstract class CoherencePolicy(val dir: DirectoryRepresentation) { +abstract class CoherencePolicy(val dir: () => DirectoryRepresentation) { def nClientStates: Int def nMasterStates: Int def nAcquireTypes: Int @@ -144,10 +146,10 @@ trait UncachedTransactions { def isUncachedReadTransaction(acq: Acquire): Bool } -abstract class CoherencePolicyWithUncached(dir: DirectoryRepresentation) extends CoherencePolicy(dir) +abstract class CoherencePolicyWithUncached(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) with UncachedTransactions -class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 2 def nMasterStates = 2 def nAcquireTypes = 6 @@ -299,7 +301,7 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncac def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 3 def nMasterStates = 2 def nAcquireTypes = 7 @@ -470,7 +472,7 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUnca def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 3 def nMasterStates = 3 def nAcquireTypes = 7 @@ -643,7 +645,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUnca def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 4 def nMasterStates = 3 def nAcquireTypes = 7 @@ -822,7 +824,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUnc def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 7 def nMasterStates = 3 def nAcquireTypes = 8 From 149d51d644d1ffb0ae7914d512f0c7b41951ff71 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 10 Jul 2014 17:10:32 -0700 Subject: [PATCH 247/374] more coherence API cleanup --- uncore/src/main/scala/cache.scala | 11 +-- uncore/src/main/scala/coherence.scala | 102 ++++++++++++++------------ uncore/src/main/scala/uncore.scala | 8 +- 3 files changed, 63 insertions(+), 58 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 686f0ac1..faa879a9 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -376,7 +376,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id - io.inner.grant.bits.payload := Grant(co.getGrantType(xact, UInt(0)), + io.inner.grant.bits.payload := Grant(co.getGrantType(xact, conf.tl.co.masterMetadataOnFlush),// TODO xact_internal.meta) xact.client_xact_id, UInt(trackerId)) @@ -454,11 +454,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact.a_type, co.masterMetadataOnFlush), - xact.addr, - UInt(trackerId)) + io.inner.probe.bits.payload := Probe(co.getProbeType(xact, /*xact_internal.meta), TODO*/ + co.masterMetadataOnFlush), + xact.addr, + UInt(trackerId)) - val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt) + val grant_type = co.getGrantType(xact, conf.tl.co.masterMetadataOnFlush)// TODO xact_internal.meta) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index abacdabd..bd6fecc3 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -28,7 +28,7 @@ object MasterMetadata { def apply(state: UInt)(implicit c: CoherencePolicy): MasterMetadata = { val m = new MasterMetadata m.state := state - m.sharers := m.sharers.flush() + m.sharers.flush() m } def apply(state: UInt, sharers: DirectoryRepresentation)(implicit c: CoherencePolicy): MasterMetadata = { @@ -51,32 +51,36 @@ class MixedMetadata(inner: CoherencePolicy, outer: CoherencePolicy) extends Cohe */ abstract class DirectoryRepresentation extends Bundle { - val sharers: UInt + val internal: UInt def pop(id: UInt): DirectoryRepresentation def push(id: UInt): DirectoryRepresentation def flush(dummy: Int = 0): DirectoryRepresentation def none(dummy: Int = 0): Bool def one(dummy: Int = 0): Bool + def count(dummy: Int = 0): UInt + def next(dummy: Int = 0): UInt } class NullRepresentation extends DirectoryRepresentation { - val sharers = UInt(0) + val internal = UInt(0) def pop(id: UInt) = this def push(id: UInt) = this def flush(dummy: Int = 0) = this def none(dummy: Int = 0) = Bool(false) def one(dummy: Int = 0) = Bool(false) + def count(dummy: Int = 0) = UInt(0) + def next(dummy: Int = 0) = UInt(0) } class FullRepresentation(nClients: Int) extends DirectoryRepresentation { - val sharers = UInt(width = nClients) - def pop(id: UInt) = { sharers := sharers & ~UIntToOH(id); this } - def push(id: UInt) = { sharers := sharers | UIntToOH(id); this } - def flush(dummy: Int = 0) = { sharers := UInt(0, width = nClients); this } - def none(dummy: Int = 0) = sharers === UInt(0) - def one(dummy: Int = 0) = PopCount(sharers) === UInt(1) - def count(dummy: Int = 0) = PopCount(sharers) - def next(dummy: Int = 0) = PriorityEncoder(sharers) + val internal = UInt(width = nClients) + def pop(id: UInt) = { internal := internal & ~UIntToOH(id); this } + def push(id: UInt) = { internal := internal | UIntToOH(id); this } + def flush(dummy: Int = 0) = { internal := UInt(0, width = nClients); this } + def none(dummy: Int = 0) = internal === UInt(0) + def one(dummy: Int = 0) = PopCount(internal) === UInt(1) + def count(dummy: Int = 0) = PopCount(internal) + def next(dummy: Int = 0) = PriorityEncoder(internal) override def clone = new FullRepresentation(nClients).asInstanceOf[this.type] } @@ -113,12 +117,14 @@ abstract class CoherencePolicy(val dir: () => DirectoryRepresentation) { def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt - def getProbeType(a_type: UInt, m: MasterMetadata): UInt + def getProbeType(a: Acquire, m: MasterMetadata): UInt def getReleaseTypeOnCacheControl(cmd: UInt): UInt def getReleaseTypeOnVoluntaryWriteback(): UInt - def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt - def getGrantType(a_type: UInt, count: UInt): UInt - def getGrantType(rel: Release, count: UInt): UInt + def getReleaseTypeOnProbe(p: Probe, m: ClientMetadata): UInt + def getGrantType(a: Acquire, m: MasterMetadata): UInt + def getGrantType(r: Release, m: MasterMetadata): UInt + //def getGrantType(a: Acquire) = getGrantType(a, new NullRepresentation) // TODO + //def getGrantType(r: Release) = getGrantType(r, new NullRepresentation) def messageHasData (rel: SourcedMessage): Bool def messageUpdatesDataArray (reply: Grant): Bool @@ -259,8 +265,8 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a_type: UInt, count: UInt): UInt = { - MuxLookup(a_type, grantReadUncached, Array( + def getGrantType(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, grantReadUncached, Array( acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -270,14 +276,14 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit )) } - def getGrantType(rel: Release, count: UInt): UInt = { - MuxLookup(rel.r_type, grantReadUncached, Array( + def getGrantType(r: Release, m: MasterMetadata): UInt = { + MuxLookup(r.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) } - def getProbeType(a_type: UInt, m: MasterMetadata): UInt = { - MuxLookup(a_type, probeCopy, Array( + def getProbeType(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, probeCopy, Array( acquireReadExclusive -> probeInvalidate, acquireReadUncached -> probeCopy, acquireWriteUncached -> probeInvalidate, @@ -427,8 +433,8 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a_type: UInt, count: UInt): UInt = { - MuxLookup(a_type, grantReadUncached, Array( + def getGrantType(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, grantReadUncached, Array( acquireReadExclusiveClean -> grantReadExclusive, acquireReadExclusiveDirty -> grantReadExclusive, acquireReadUncached -> grantReadUncached, @@ -438,15 +444,15 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi acquireAtomicUncached -> grantAtomicUncached )) } - def getGrantType(rel: Release, count: UInt): UInt = { - MuxLookup(rel.r_type, grantReadUncached, Array( + def getGrantType(r: Release, m: MasterMetadata): UInt = { + MuxLookup(r.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) } - def getProbeType(a_type: UInt, m: MasterMetadata): UInt = { - MuxLookup(a_type, probeCopy, Array( + def getProbeType(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, probeCopy, Array( acquireReadExclusiveClean -> probeInvalidate, acquireReadExclusiveDirty -> probeInvalidate, acquireReadUncached -> probeCopy, @@ -604,9 +610,9 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a_type: UInt, count: UInt): UInt = { - MuxLookup(a_type, grantReadUncached, Array( - acquireReadShared -> Mux(count > UInt(0), grantReadShared, grantReadExclusive), + def getGrantType(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, grantReadUncached, Array( + acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -615,14 +621,14 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi acquireAtomicUncached -> grantAtomicUncached )) } - def getGrantType(rel: Release, count: UInt): UInt = { - MuxLookup(rel.r_type, grantReadUncached, Array( + def getGrantType(r: Release, m: MasterMetadata): UInt = { + MuxLookup(r.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) } - def getProbeType(a_type: UInt, m: MasterMetadata): UInt = { - MuxLookup(a_type, probeCopy, Array( + def getProbeType(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, probeCopy, Array( acquireReadShared -> probeDowngrade, acquireReadExclusive -> probeInvalidate, acquireReadUncached -> probeCopy, @@ -778,9 +784,9 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a_type: UInt, count: UInt): UInt = { - MuxLookup(a_type, grantReadUncached, Array( - acquireReadShared -> Mux(count > UInt(0), grantReadShared, grantReadExclusive), + def getGrantType(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, grantReadUncached, Array( + acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -789,15 +795,15 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW acquireAtomicUncached -> grantAtomicUncached )) } - def getGrantType(rel: Release, count: UInt): UInt = { - MuxLookup(rel.r_type, grantReadUncached, Array( + def getGrantType(r: Release, m: MasterMetadata): UInt = { + MuxLookup(r.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) } - def getProbeType(a_type: UInt, m: MasterMetadata): UInt = { - MuxLookup(a_type, probeCopy, Array( + def getProbeType(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, probeCopy, Array( acquireReadShared -> probeDowngrade, acquireReadExclusive -> probeInvalidate, acquireReadUncached -> probeCopy, @@ -975,9 +981,9 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a_type: UInt, count: UInt): UInt = { - MuxLookup(a_type, grantReadUncached, Array( - acquireReadShared -> Mux(count > UInt(0), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? + def getGrantType(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, grantReadUncached, Array( + acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -987,15 +993,15 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI? )) } - def getGrantType(rel: Release, count: UInt): UInt = { - MuxLookup(rel.r_type, grantReadUncached, Array( + def getGrantType(r: Release, m: MasterMetadata): UInt = { + MuxLookup(r.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) } - def getProbeType(a_type: UInt, m: MasterMetadata): UInt = { - MuxLookup(a_type, probeCopy, Array( + def getProbeType(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, probeCopy, Array( acquireReadShared -> probeDowngrade, acquireReadExclusive -> probeInvalidate, acquireReadUncached -> probeCopy, diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 8f1a7a67..2e62d79f 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -130,7 +130,7 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends XactTracker { io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id - io.inner.grant.bits.payload := Grant(co.getGrantType(xact, UInt(0)), + io.inner.grant.bits.payload := Grant(co.getGrantType(xact, co.masterMetadataOnFlush), xact.client_xact_id, UInt(trackerId)) @@ -161,7 +161,6 @@ class AcquireTracker(trackerId: Int, bankId: Int) extends XactTracker { val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) //TODO: Will need id reg for merged release xacts - val init_sharer_cnt = Reg(init=UInt(0, width = log2Up(nClients))) val release_count = if (nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(nClients))) val probe_flags = Reg(init=Bits(0, width = nClients)) val curr_p_id = PriorityEncoder(probe_flags) @@ -195,11 +194,11 @@ class AcquireTracker(trackerId: Int, bankId: Int) extends XactTracker { io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact.a_type, co.masterMetadataOnFlush), + io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.masterMetadataOnFlush), xact.addr, UInt(trackerId)) - val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt) + val grant_type = co.getGrantType(xact, co.masterMetadataOnFlush) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id @@ -219,7 +218,6 @@ class AcquireTracker(trackerId: Int, bankId: Int) extends XactTracker { when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src - init_sharer_cnt := UInt(nClients) // TODO: Broadcast only probe_flags := probe_initial_flags if(nClients > 1) { release_count := PopCount(probe_initial_flags) From 53b8d7b03164a97c5bb03cd6910b978168cf1252 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 13 Jul 2014 22:29:15 -0700 Subject: [PATCH 248/374] use new coherence methods in l2, ready to query dir logic --- uncore/src/main/scala/cache.scala | 83 ++++++++++++++++++++------- uncore/src/main/scala/coherence.scala | 2 +- 2 files changed, 62 insertions(+), 23 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index faa879a9..3de8ba4a 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -109,7 +109,7 @@ trait HasL2Id extends Bundle with CoherenceAgentParameters { trait HasL2InternalRequestState extends L2HellaCacheBundle { val tag_match = Bool() - val old_meta = new L2Metadata + val meta = new L2Metadata val way_en = Bits(width = nWays) } @@ -167,7 +167,7 @@ class L2MetadataArray extends L2HellaCacheModule { io.resp.valid := Reg(next = s1_clk_en) io.resp.bits.id := RegEnable(s1_id, s1_clk_en) io.resp.bits.tag_match := s2_tag_match - io.resp.bits.old_meta := Mux(s2_tag_match, + io.resp.bits.meta := Mux(s2_tag_match, L2Metadata(s2_repl_meta.tag, s2_hit_coh), s2_repl_meta) io.resp.bits.way_en := Mux(s2_tag_match, s2_tag_match_way, s2_replaced_way_en) @@ -354,18 +354,16 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) val xact = Reg{ new Release } + val xact_internal = Reg{ new L2MetaResp } val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) - val new_meta = Reg(new L2Metadata) - val incoming_rel = io.inner.release.bits io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, incoming_rel.payload.addr) && + io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) io.outer.grant.ready := Bool(false) io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.header.src := UInt(bankId) - //io.outer.acquire.bits.header.dst TODO io.outer.acquire.bits.payload := Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId), @@ -376,32 +374,47 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id - io.inner.grant.bits.payload := Grant(co.getGrantType(xact, conf.tl.co.masterMetadataOnFlush),// TODO xact_internal.meta) + io.inner.grant.bits.payload := Grant(co.getGrantType(xact, co.masterMetadataOnFlush),// TODO xact_internal.meta) xact.client_xact_id, UInt(trackerId)) + io.data_read.valid := Bool(false) + io.data_write.valid := Bool(false) + io.data_write.bits.id := UInt(trackerId) + io.data_write.bits.way_en := xact_internal.way_en + io.data_write.bits.addr := xact.addr + io.data_write.bits.wmask := SInt(-1) + io.data_write.bits.data := xact.data + io.meta_read.valid := Bool(false) + io.meta_read.bits.id := UInt(trackerId) + io.meta_read.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta_read.bits.tag := xact.addr >> UInt(untagBits) + io.meta_write.valid := Bool(false) + io.meta_write.bits.id := UInt(trackerId) + io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta_write.bits.way_en := xact_internal.way_en + io.meta_write.bits.data := xact_internal.meta + + when(io.meta_resp.valid) { xact_internal := io.meta_resp.bits } switch (state) { is(s_idle) { io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { - xact := incoming_rel.payload - init_client_id := incoming_rel.header.src + xact := c_rel.payload + init_client_id := c_rel.header.src state := s_mem } - } -/* + }/* is(s_meta_read) { when(io.meta_read.ready) state := s_meta_resp } is(s_meta_resp) { when(io.meta_resp.valid) { - new_meta := L2Metadata(io.meta.resp.bits.old_meta.tag, io.meta.resp.bits.old_meta.sharers, io.meta.resp.bits - old_meta := io.meta.resp.bits.old_meta + xact_internal.meta := tl.co.masterMetadataOnRelease(xact, xact_internal.meta, init_client_id)) state := Mux(s_meta_write -Mux(co.messageHasData(xact), s_mem, s_ack) - } - */ + Mux(co.messageHasData(xact), s_mem, s_ack) + }*/ is(s_mem) { io.outer.acquire.valid := Bool(true) when(io.outer.acquire.ready) { state := s_ack } @@ -417,11 +430,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) val state = Reg(init=s_idle) val xact = Reg{ new Acquire } + val xact_internal = Reg{ new L2MetaResp } val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) //TODO: Will need id reg for merged release xacts - - val init_sharer_cnt = Reg(init=UInt(0, width = log2Up(nClients))) - val release_count = if(nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(nClients))) + + val release_count = if (nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(nClients))) val probe_flags = Reg(init=Bits(0, width = nClients)) val curr_p_id = PriorityEncoder(probe_flags) @@ -447,7 +460,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.header.src := UInt(bankId) - //io.outer.acquire.bits.header.dst TODO io.outer.acquire.bits.payload := outer_read io.outer.grant.ready := io.inner.grant.ready @@ -459,7 +471,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { xact.addr, UInt(trackerId)) - val grant_type = co.getGrantType(xact, conf.tl.co.masterMetadataOnFlush)// TODO xact_internal.meta) + val grant_type = co.getGrantType(xact, co.masterMetadataOnFlush)// TODO xact_internal.meta) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id @@ -471,6 +483,28 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) + io.data_read.valid := Bool(false) + io.data_read.bits.id := UInt(trackerId) + io.data_read.bits.way_en := xact_internal.way_en + io.data_read.bits.addr := xact.addr + io.data_write.valid := Bool(false) + io.data_write.bits.id := UInt(trackerId) + io.data_write.bits.way_en := xact_internal.way_en + io.data_write.bits.addr := xact.addr + io.data_write.bits.wmask := SInt(-1) + io.data_write.bits.data := xact.data + io.meta_read.valid := Bool(false) + io.meta_read.bits.id := UInt(trackerId) + io.meta_read.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta_read.bits.tag := xact.addr >> UInt(untagBits) + io.meta_write.valid := Bool(false) + io.meta_write.bits.id := UInt(trackerId) + io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta_write.bits.way_en := xact_internal.way_en + io.meta_write.bits.data := xact_internal.meta + + when(io.meta_resp.valid && io.meta_resp.bits.id === UInt(trackerId)) { xact_internal := io.meta_resp.bits } + switch (state) { is(s_idle) { io.inner.acquire.ready := Bool(true) @@ -479,7 +513,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src - init_sharer_cnt := UInt(nClients) // TODO: Broadcast only probe_flags := probe_initial_flags if(nClients > 1) { release_count := PopCount(probe_initial_flags) @@ -490,6 +523,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { Mux(needs_outer_read, s_mem_read, s_make_grant)) } } + // s_read_meta + // xact_internal := resp is(s_probe) { // Generate probes io.inner.probe.valid := probe_flags.orR @@ -499,6 +534,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { // Handle releases, which may have data to be written back when(io.inner.release.valid) { + //xact_internal.meta.coh := tl.co.masterMetadataOnRelease( + // io.inner.release.bits.payload, + // xact_internal.meta.coh, + // init_client_id) when(co.messageHasData(c_rel.payload)) { io.outer.acquire.valid := Bool(true) io.outer.acquire.bits.payload := outer_write_rel diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index bd6fecc3..7dbff4aa 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -74,7 +74,7 @@ class NullRepresentation extends DirectoryRepresentation { class FullRepresentation(nClients: Int) extends DirectoryRepresentation { val internal = UInt(width = nClients) - def pop(id: UInt) = { internal := internal & ~UIntToOH(id); this } + def pop(id: UInt) = { internal := internal & ~UIntToOH(id); this } // make new FullRep to return? def push(id: UInt) = { internal := internal | UIntToOH(id); this } def flush(dummy: Int = 0) = { internal := UInt(0, width = nClients); this } def none(dummy: Int = 0) = internal === UInt(0) From 82fe22f958a812cf8ff1d7755d5d4cb01cb7c242 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 24 Sep 2014 11:14:36 -0700 Subject: [PATCH 249/374] support for multiple tilelink paramerterizations in same design Conflicts: src/main/scala/cache.scala --- uncore/src/main/scala/cache.scala | 46 ++++++++++++++++------------ uncore/src/main/scala/tilelink.scala | 5 +-- uncore/src/main/scala/uncore.scala | 44 ++++++++++++++------------ 3 files changed, 54 insertions(+), 41 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 3de8ba4a..ca6b5c3f 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -212,13 +212,14 @@ class L2DataArray extends L2HellaCacheModule { io.write.ready := Bool(true) } -class L2HellaCache(bankId: Int) extends CoherenceAgent with L2HellaCacheParameters { +class L2HellaCache(bankId: Int, innerId: String, outerId: String) extends + CoherenceAgent(innerId, outerId) with L2HellaCacheParameters { require(isPow2(nSets)) require(isPow2(nWays)) require(refillCycles == 1) - val tshrfile = Module(new TSHRFile(bankId)) + val tshrfile = Module(new TSHRFile(bankId, innerId, outerId)) val meta = Module(new L2MetadataArray) val data = Module(new L2DataArray) @@ -234,10 +235,10 @@ class L2HellaCache(bankId: Int) extends CoherenceAgent with L2HellaCacheParamete } -class TSHRFile(bankId: Int) extends L2HellaCacheModule { +class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCacheModule { val io = new Bundle { - val inner = (new TileLinkIO).flip - val outer = new UncachedTileLinkIO + val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip + val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) val incoherent = Vec.fill(nClients){Bool()}.asInput val meta_read = Decoupled(new L2MetaReadReq) val meta_write = Decoupled(new L2MetaWriteReq) @@ -261,9 +262,9 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule { // Create TSHRs for outstanding transactions val trackerList = (0 until nReleaseTransactors).map { id => - Module(new L2VoluntaryReleaseTracker(id, bankId)) + Module(new L2VoluntaryReleaseTracker(id, bankId, innerId, outerId)) } ++ (nReleaseTransactors until nTransactors).map { id => - Module(new L2AcquireTracker(id, bankId)) + Module(new L2AcquireTracker(id, bankId, innerId, outerId)) } // Propagate incoherence flags @@ -312,7 +313,8 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule { ack.ready := Bool(true) // Arbitrate for the outer memory port - val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size)) + val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), + {case TLId => outerId}) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } io.outer <> outer_arb.io.out @@ -327,10 +329,10 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule { } -abstract class L2XactTracker extends L2HellaCacheModule { +abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCacheModule { val io = new Bundle { - val inner = (new TileLinkIO).flip - val outer = new UncachedTileLinkIO + val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip + val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) val tile_incoherent = Bits(INPUT, nClients) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) @@ -350,7 +352,7 @@ abstract class L2XactTracker extends L2HellaCacheModule { } -class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTracker { +class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) val xact = Reg{ new Release } @@ -364,10 +366,11 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack io.outer.grant.ready := Bool(false) io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.header.src := UInt(bankId) - io.outer.acquire.bits.payload := Acquire(co.getUncachedWriteAcquireType, + io.outer.acquire.bits.payload := Bundle(Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId), - xact.data) + xact.data), + { case TLId => outerId }) io.inner.acquire.ready := Bool(false) io.inner.probe.valid := Bool(false) io.inner.release.ready := Bool(false) @@ -426,7 +429,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack } } -class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { +class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) val state = Reg(init=s_idle) val xact = Reg{ new Acquire } @@ -440,11 +443,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val pending_outer_write = co.messageHasData(xact) val pending_outer_read = co.requiresOuterRead(xact.a_type) - val outer_write_acq = Acquire(co.getUncachedWriteAcquireType, - xact.addr, UInt(trackerId), xact.data) - val outer_write_rel = Acquire(co.getUncachedWriteAcquireType, - xact.addr, UInt(trackerId), c_rel.payload.data) - val outer_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) + val outer_write_acq = Bundle(Acquire(co.getUncachedWriteAcquireType, + xact.addr, UInt(trackerId), xact.data), + { case TLId => outerId }) + val outer_write_rel = Bundle(Acquire(co.getUncachedWriteAcquireType, + xact.addr, UInt(trackerId), c_rel.payload.data), + { case TLId => outerId }) + val outer_read = Bundle(Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)), + { case TLId => outerId }) val probe_initial_flags = Bits(width = nClients) probe_initial_flags := Bits(0) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index af736c97..8bb98f0a 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -3,6 +3,7 @@ package uncore import Chisel._ +case object TLId extends Field[String] case object TLCoherence extends Field[CoherencePolicyWithUncached] case object TLAddrBits extends Field[Int] case object TLMasterXactIdBits extends Field[Int] @@ -155,13 +156,13 @@ class Grant extends MasterSourcedMessage class Finish extends ClientSourcedMessage with HasMasterTransactionId -class UncachedTileLinkIO extends Bundle { +class UncachedTileLinkIO(p: Option[Parameters] = None) extends Bundle()(p) { val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) } -class TileLinkIO extends UncachedTileLinkIO { +class TileLinkIO(p: Option[Parameters] = None) extends UncachedTileLinkIO(p) { val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip val release = new DecoupledIO(new LogicalNetworkIO(new Release)) } diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 2e62d79f..37e535f8 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -15,22 +15,23 @@ abstract trait CoherenceAgentParameters extends UsesParameters { val nClients = params(NClients) } -abstract class CoherenceAgent extends Module +abstract class CoherenceAgent(innerId: String, outerId: String) extends Module with CoherenceAgentParameters { val io = new Bundle { - val inner = (new TileLinkIO).flip - val outer = new UncachedTileLinkIO + val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip + val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) val incoherent = Vec.fill(nClients){Bool()}.asInput } } -class L2CoherenceAgent(bankId: Int) extends CoherenceAgent { +class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends + CoherenceAgent(innerId, outerId) { // Create SHRs for outstanding transactions val trackerList = (0 until nReleaseTransactors).map(id => - Module(new VoluntaryReleaseTracker(id, bankId))) ++ + Module(new VoluntaryReleaseTracker(id, bankId, innerId, outerId))) ++ (nReleaseTransactors until nTransactors).map(id => - Module(new AcquireTracker(id, bankId))) + Module(new AcquireTracker(id, bankId, innerId, outerId))) // Propagate incoherence flags trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) @@ -82,17 +83,18 @@ class L2CoherenceAgent(bankId: Int) extends CoherenceAgent { ack.ready := Bool(true) // Create an arbiter for the one memory port - val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size)) + val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), + {case TLId => outerId}) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } io.outer <> outer_arb.io.out } -abstract class XactTracker extends Module { +abstract class XactTracker(innerId: String, outerId: String) extends Module { val (co, nClients) = (params(TLCoherence),params(NClients)) val io = new Bundle { - val inner = (new TileLinkIO).flip - val outer = new UncachedTileLinkIO + val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip + val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) val tile_incoherent = Bits(INPUT, params(NClients)) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) @@ -105,7 +107,7 @@ abstract class XactTracker extends Module { val m_gnt = io.outer.grant.bits } -class VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends XactTracker { +class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) val xact = Reg{ new Release } @@ -120,10 +122,11 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends XactTracker { io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.header.src := UInt(bankId) //io.outer.acquire.bits.header.dst TODO - io.outer.acquire.bits.payload := Acquire(co.getUncachedWriteAcquireType, + io.outer.acquire.bits.payload := Bundle(Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId), - xact.data) + xact.data), + { case TLId => outerId }) io.inner.acquire.ready := Bool(false) io.inner.probe.valid := Bool(false) io.inner.release.ready := Bool(false) @@ -154,7 +157,7 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends XactTracker { } } -class AcquireTracker(trackerId: Int, bankId: Int) extends XactTracker { +class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) val state = Reg(init=s_idle) val xact = Reg{ new Acquire } @@ -167,11 +170,14 @@ class AcquireTracker(trackerId: Int, bankId: Int) extends XactTracker { val pending_outer_write = co.messageHasData(xact) val pending_outer_read = co.requiresOuterRead(xact.a_type) - val outer_write_acq = Acquire(co.getUncachedWriteAcquireType, - xact.addr, UInt(trackerId), xact.data) - val outer_write_rel = Acquire(co.getUncachedWriteAcquireType, - xact.addr, UInt(trackerId), c_rel.payload.data) - val outer_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) + val outer_write_acq = Bundle(Acquire(co.getUncachedWriteAcquireType, + xact.addr, UInt(trackerId), xact.data), + { case TLId => outerId }) + val outer_write_rel = Bundle(Acquire(co.getUncachedWriteAcquireType, + xact.addr, UInt(trackerId), c_rel.payload.data), + { case TLId => outerId }) + val outer_read = Bundle(Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)), + { case TLId => outerId }) val probe_initial_flags = Bits(width = nClients) probe_initial_flags := Bits(0) From 7571695320f3804387a87aec570cf7ba80c01da6 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 24 Sep 2014 15:11:24 -0700 Subject: [PATCH 250/374] Removed broken or unfinished modules, new MemPipeIO converter --- uncore/src/main/scala/bigmem.scala | 80 ++++ uncore/src/main/scala/cache.scala | 499 ----------------------- uncore/src/main/scala/llc.scala | 554 -------------------------- uncore/src/main/scala/memserdes.scala | 109 +++++ 4 files changed, 189 insertions(+), 1053 deletions(-) create mode 100644 uncore/src/main/scala/bigmem.scala delete mode 100644 uncore/src/main/scala/llc.scala diff --git a/uncore/src/main/scala/bigmem.scala b/uncore/src/main/scala/bigmem.scala new file mode 100644 index 00000000..fbbc4c07 --- /dev/null +++ b/uncore/src/main/scala/bigmem.scala @@ -0,0 +1,80 @@ +// See LICENSE for license details. + +package uncore +import Chisel._ + +class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UInt], noMask: Boolean = false)(gen: => T) extends Module +{ + class Inputs extends Bundle { + val addr = UInt(INPUT, log2Up(n)) + val rw = Bool(INPUT) + val wdata = gen.asInput + val wmask = gen.asInput + override def clone = new Inputs().asInstanceOf[this.type] + } + val io = new Bundle { + val in = Valid(new Inputs).flip + val rdata = gen.asOutput + } + val data = gen + val colMux = if (2*data.getWidth <= leaf.data.getWidth && n > leaf.n) 1 << math.floor(math.log(leaf.data.getWidth/data.getWidth)/math.log(2)).toInt else 1 + val nWide = if (data.getWidth > leaf.data.getWidth) 1+(data.getWidth-1)/leaf.data.getWidth else 1 + val nDeep = if (n > colMux*leaf.n) 1+(n-1)/(colMux*leaf.n) else 1 + if (nDeep > 1 || colMux > 1) + require(isPow2(n) && isPow2(leaf.n)) + + val rdataDeep = Vec.fill(nDeep){Bits()} + val rdataSel = Vec.fill(nDeep){Bool()} + for (i <- 0 until nDeep) { + val in = Pipe(io.in.valid && (if (nDeep == 1) Bool(true) else UInt(i) === io.in.bits.addr(log2Up(n)-1, log2Up(n/nDeep))), io.in.bits, preLatency) + val idx = in.bits.addr(log2Up(n/nDeep/colMux)-1, 0) + val wdata = in.bits.wdata.toBits + val wmask = in.bits.wmask.toBits + val ren = in.valid && !in.bits.rw + val reg_ren = Reg(next=ren) + val rdata = Vec.fill(nWide){Bits()} + + val r = Pipe(ren, in.bits.addr, postLatency) + + for (j <- 0 until nWide) { + val mem = leaf.clone + var dout: Bits = null + val ridx = if (postLatency > 0) Reg(Bits()) else null + + var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.getWidth*(j+1))-1, leaf.data.getWidth*j)) + if (colMux > 1) + wmask0 = wmask0 & FillInterleaved(gen.getWidth, UIntToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) + val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.getWidth*(j+1))-1, leaf.data.getWidth*j)) + when (in.valid) { + when (in.bits.rw) { + if (noMask) + mem.write(idx, wdata0) + else + mem.write(idx, wdata0, wmask0) + } + .otherwise { if (postLatency > 0) ridx := idx } + } + + if (postLatency == 0) { + dout = mem(idx) + } else if (postLatency == 1) { + dout = mem(ridx) + } else + dout = Pipe(reg_ren, mem(ridx), postLatency-1).bits + + rdata(j) := dout + } + val rdataWide = rdata.reduceLeft((x, y) => Cat(y, x)) + + var colMuxOut = rdataWide + if (colMux > 1) { + val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.getWidth*(k+1)-1, gen.getWidth*k))) + colMuxOut = colMuxIn(r.bits(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) + } + + rdataDeep(i) := colMuxOut + rdataSel(i) := r.valid + } + + io.rdata := Mux1H(rdataSel, rdataDeep) +} diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index ca6b5c3f..26157a75 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -96,502 +96,3 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { io.read.ready := !rst && !io.write.valid // so really this could be a 6T RAM io.write.ready := !rst } - -abstract trait L2HellaCacheParameters extends CacheParameters - with CoherenceAgentParameters - -abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters -abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters - -trait HasL2Id extends Bundle with CoherenceAgentParameters { - val id = UInt(width = log2Up(nTransactors)) -} - -trait HasL2InternalRequestState extends L2HellaCacheBundle { - val tag_match = Bool() - val meta = new L2Metadata - val way_en = Bits(width = nWays) -} - -object L2Metadata { - def apply(tag: Bits, coh: MasterMetadata) = { - val meta = new L2Metadata - meta.tag := tag - meta.coh := coh - meta - } -} -class L2Metadata extends Metadata with L2HellaCacheParameters { - val coh = co.masterMetadataOnFlush.clone -} - -class L2MetaReadReq extends MetaReadReq with HasL2Id { - val tag = Bits(width = tagBits) -} - -class L2MetaWriteReq extends MetaWriteReq[L2Metadata](new L2Metadata) - with HasL2Id - -class L2MetaResp extends L2HellaCacheBundle - with HasL2Id - with HasL2InternalRequestState - -class L2MetadataArray extends L2HellaCacheModule { - val io = new Bundle { - val read = Decoupled(new L2MetaReadReq).flip - val write = Decoupled(new L2MetaWriteReq).flip - val resp = Valid(new L2MetaResp) - } - - val meta = Module(new MetadataArray(() => L2Metadata(UInt(0), co.masterMetadataOnFlush))) - meta.io.read <> io.read - meta.io.write <> io.write - - val s1_clk_en = Reg(next = io.read.fire()) - val s1_tag = RegEnable(io.read.bits.tag, io.read.valid) - val s1_id = RegEnable(io.read.bits.id, io.read.valid) - def wayMap[T <: Data](f: Int => T) = Vec((0 until nWays).map(f)) - val s1_tag_eq_way = wayMap((w: Int) => meta.io.resp(w).tag === s1_tag) - val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && co.isValid(meta.io.resp(w).coh)).toBits - val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) - val s2_tag_match = s2_tag_match_way.orR - val s2_hit_coh = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).coh, s1_clk_en))) - //val s2_hit = s2_tag_match && tl.co.isHit(s2_req.cmd, s2_hit_state) && s2_hit_state === tl.co.newStateOnHit(s2_req.cmd, s2_hit_state) - - val replacer = params(Replacer)() - val s1_replaced_way_en = UIntToOH(replacer.way) - val s2_replaced_way_en = UIntToOH(RegEnable(replacer.way, s1_clk_en)) - val s2_repl_meta = Mux1H(s2_replaced_way_en, wayMap((w: Int) => - RegEnable(meta.io.resp(w), s1_clk_en && s1_replaced_way_en(w))).toSeq) - - io.resp.valid := Reg(next = s1_clk_en) - io.resp.bits.id := RegEnable(s1_id, s1_clk_en) - io.resp.bits.tag_match := s2_tag_match - io.resp.bits.meta := Mux(s2_tag_match, - L2Metadata(s2_repl_meta.tag, s2_hit_coh), - s2_repl_meta) - io.resp.bits.way_en := Mux(s2_tag_match, s2_tag_match_way, s2_replaced_way_en) -} - -class L2DataReadReq extends L2HellaCacheBundle with HasL2Id { - val way_en = Bits(width = nWays) - val addr = Bits(width = params(TLAddrBits)) -} - -class L2DataWriteReq extends L2DataReadReq { - val wmask = Bits(width = params(TLWriteMaskBits)) - val data = Bits(width = params(TLDataBits)) -} - -class L2DataResp extends Bundle with HasL2Id { - val data = Bits(width = params(TLDataBits)) -} - -class L2DataArray extends L2HellaCacheModule { - val io = new Bundle { - val read = Decoupled(new L2DataReadReq).flip - val write = Decoupled(new L2DataWriteReq).flip - val resp = Valid(new L2DataResp) - } - - val waddr = io.write.bits.addr - val raddr = io.read.bits.addr - val wmask = FillInterleaved(wordBits, io.write.bits.wmask) - val resp = (0 until nWays).map { w => - val array = Mem(Bits(width=params(RowBits)), nSets*refillCycles, seqRead = true) - when (io.write.bits.way_en(w) && io.write.valid) { - array.write(waddr, io.write.bits.data, wmask) - } - array(RegEnable(raddr, io.read.bits.way_en(w) && io.read.valid)) - } - io.resp.valid := ShiftRegister(io.read.valid, 2) - io.resp.bits.id := ShiftRegister(io.read.bits.id, 2) - io.resp.bits.data := Mux1H(ShiftRegister(io.read.bits.way_en, 2), resp) - - io.read.ready := Bool(true) - io.write.ready := Bool(true) -} - -class L2HellaCache(bankId: Int, innerId: String, outerId: String) extends - CoherenceAgent(innerId, outerId) with L2HellaCacheParameters { - - require(isPow2(nSets)) - require(isPow2(nWays)) - require(refillCycles == 1) - - val tshrfile = Module(new TSHRFile(bankId, innerId, outerId)) - val meta = Module(new L2MetadataArray) - val data = Module(new L2DataArray) - - tshrfile.io.inner <> io.inner - tshrfile.io.meta_read <> meta.io.read - tshrfile.io.meta_write <> meta.io.write - tshrfile.io.meta_resp <> meta.io.resp - tshrfile.io.data_read <> data.io.read - tshrfile.io.data_write <> data.io.write - tshrfile.io.data_resp <> data.io.resp - io.outer <> tshrfile.io.outer - io.incoherent <> tshrfile.io.incoherent -} - - -class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCacheModule { - val io = new Bundle { - val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip - val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) - val incoherent = Vec.fill(nClients){Bool()}.asInput - val meta_read = Decoupled(new L2MetaReadReq) - val meta_write = Decoupled(new L2MetaWriteReq) - val meta_resp = Valid(new L2MetaResp).flip - val data_read = Decoupled(new L2DataReadReq) - val data_write = Decoupled(new L2DataWriteReq) - val data_resp = Valid(new L2DataResp).flip - } - - // Wiring helper funcs - def doOutputArbitration[T <: Data](out: DecoupledIO[T], ins: Seq[DecoupledIO[T]]) { - val arb = Module(new RRArbiter(out.bits.clone, ins.size)) - out <> arb.io.out - arb.io.in zip ins map { case (a, in) => a <> in } - } - - def doInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { - outs.map(_.bits := in.bits) - outs.zipWithIndex.map { case (o, i) => o.valid := UInt(i) === in.bits.id } - } - - // Create TSHRs for outstanding transactions - val trackerList = (0 until nReleaseTransactors).map { id => - Module(new L2VoluntaryReleaseTracker(id, bankId, innerId, outerId)) - } ++ (nReleaseTransactors until nTransactors).map { id => - Module(new L2AcquireTracker(id, bankId, innerId, outerId)) - } - - // Propagate incoherence flags - trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) - - // Handle acquire transaction initiation - val acquire = io.inner.acquire - val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - val block_acquires = any_acquire_conflict - - val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) - for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.inner - alloc_arb.io.in(i).valid := t.acquire.ready - t.acquire.bits := acquire.bits - t.acquire.valid := alloc_arb.io.in(i).ready - } - acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && !block_acquires - alloc_arb.io.out.ready := acquire.valid && !block_acquires - - // Handle probe requests - doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) - - // Handle releases, which might be voluntary and might have data - val release = io.inner.release - val voluntary = co.isVoluntary(release.bits.payload) - val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) - val block_releases = Bool(false) - val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b} - //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response - val release_idx = Mux(voluntary, UInt(0), release.bits.payload.master_xact_id) - for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.inner - t.release.bits := release.bits - t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases - } - release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases - - // Reply to initial requestor - doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) - - // Free finished transactions - val ack = io.inner.finish - trackerList.map(_.io.inner.finish.valid := ack.valid) - trackerList.map(_.io.inner.finish.bits := ack.bits) - ack.ready := Bool(true) - - // Arbitrate for the outer memory port - val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), - {case TLId => outerId}) - outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } - io.outer <> outer_arb.io.out - - // Local memory - doOutputArbitration(io.meta_read, trackerList.map(_.io.meta_read)) - doOutputArbitration(io.meta_write, trackerList.map(_.io.meta_write)) - doOutputArbitration(io.data_read, trackerList.map(_.io.data_read)) - doOutputArbitration(io.data_write, trackerList.map(_.io.data_write)) - doInputRouting(io.meta_resp, trackerList.map(_.io.meta_resp)) - doInputRouting(io.data_resp, trackerList.map(_.io.data_resp)) - -} - - -abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCacheModule { - val io = new Bundle { - val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip - val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) - val tile_incoherent = Bits(INPUT, nClients) - val has_acquire_conflict = Bool(OUTPUT) - val has_release_conflict = Bool(OUTPUT) - val meta_read = Decoupled(new L2MetaReadReq) - val meta_write = Decoupled(new L2MetaWriteReq) - val meta_resp = Valid(new L2MetaResp).flip - val data_read = Decoupled(new L2DataReadReq) - val data_write = Decoupled(new L2DataWriteReq) - val data_resp = Valid(new L2DataResp).flip - } - - val c_acq = io.inner.acquire.bits - val c_rel = io.inner.release.bits - val c_gnt = io.inner.grant.bits - val c_ack = io.inner.finish.bits - val m_gnt = io.outer.grant.bits - -} - -class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) - val state = Reg(init=s_idle) - val xact = Reg{ new Release } - val xact_internal = Reg{ new L2MetaResp } - val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) - - io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && - (state != s_idle) - - io.outer.grant.ready := Bool(false) - io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits.header.src := UInt(bankId) - io.outer.acquire.bits.payload := Bundle(Acquire(co.getUncachedWriteAcquireType, - xact.addr, - UInt(trackerId), - xact.data), - { case TLId => outerId }) - io.inner.acquire.ready := Bool(false) - io.inner.probe.valid := Bool(false) - io.inner.release.ready := Bool(false) - io.inner.grant.valid := Bool(false) - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := init_client_id - io.inner.grant.bits.payload := Grant(co.getGrantType(xact, co.masterMetadataOnFlush),// TODO xact_internal.meta) - xact.client_xact_id, - UInt(trackerId)) - - io.data_read.valid := Bool(false) - io.data_write.valid := Bool(false) - io.data_write.bits.id := UInt(trackerId) - io.data_write.bits.way_en := xact_internal.way_en - io.data_write.bits.addr := xact.addr - io.data_write.bits.wmask := SInt(-1) - io.data_write.bits.data := xact.data - io.meta_read.valid := Bool(false) - io.meta_read.bits.id := UInt(trackerId) - io.meta_read.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta_read.bits.tag := xact.addr >> UInt(untagBits) - io.meta_write.valid := Bool(false) - io.meta_write.bits.id := UInt(trackerId) - io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta_write.bits.way_en := xact_internal.way_en - io.meta_write.bits.data := xact_internal.meta - - when(io.meta_resp.valid) { xact_internal := io.meta_resp.bits } - - switch (state) { - is(s_idle) { - io.inner.release.ready := Bool(true) - when( io.inner.release.valid ) { - xact := c_rel.payload - init_client_id := c_rel.header.src - state := s_mem - } - }/* - is(s_meta_read) { - when(io.meta_read.ready) state := s_meta_resp - } - is(s_meta_resp) { - when(io.meta_resp.valid) { - xact_internal.meta := tl.co.masterMetadataOnRelease(xact, xact_internal.meta, init_client_id)) - state := Mux(s_meta_write - Mux(co.messageHasData(xact), s_mem, s_ack) - }*/ - is(s_mem) { - io.outer.acquire.valid := Bool(true) - when(io.outer.acquire.ready) { state := s_ack } - } - is(s_ack) { - io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { state := s_idle } - } - } -} - -class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) - val state = Reg(init=s_idle) - val xact = Reg{ new Acquire } - val xact_internal = Reg{ new L2MetaResp } - val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) - //TODO: Will need id reg for merged release xacts - - val release_count = if (nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(nClients))) - val probe_flags = Reg(init=Bits(0, width = nClients)) - val curr_p_id = PriorityEncoder(probe_flags) - - val pending_outer_write = co.messageHasData(xact) - val pending_outer_read = co.requiresOuterRead(xact.a_type) - val outer_write_acq = Bundle(Acquire(co.getUncachedWriteAcquireType, - xact.addr, UInt(trackerId), xact.data), - { case TLId => outerId }) - val outer_write_rel = Bundle(Acquire(co.getUncachedWriteAcquireType, - xact.addr, UInt(trackerId), c_rel.payload.data), - { case TLId => outerId }) - val outer_read = Bundle(Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)), - { case TLId => outerId }) - - val probe_initial_flags = Bits(width = nClients) - probe_initial_flags := Bits(0) - if (nClients > 1) { - // issue self-probes for uncached read xacts to facilitate I$ coherence - val probe_self = Bool(true) //co.needsSelfProbe(io.inner.acquire.bits.payload) - val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(nClients)-1,0))) - probe_initial_flags := ~(io.tile_incoherent | myflag) - } - - io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) - - io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits.header.src := UInt(bankId) - io.outer.acquire.bits.payload := outer_read - io.outer.grant.ready := io.inner.grant.ready - - io.inner.probe.valid := Bool(false) - io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact, /*xact_internal.meta), TODO*/ - co.masterMetadataOnFlush), - xact.addr, - UInt(trackerId)) - - val grant_type = co.getGrantType(xact, co.masterMetadataOnFlush)// TODO xact_internal.meta) - io.inner.grant.valid := Bool(false) - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := init_client_id - io.inner.grant.bits.payload := Grant(grant_type, - xact.client_xact_id, - UInt(trackerId), - m_gnt.payload.data) - - io.inner.acquire.ready := Bool(false) - io.inner.release.ready := Bool(false) - - io.data_read.valid := Bool(false) - io.data_read.bits.id := UInt(trackerId) - io.data_read.bits.way_en := xact_internal.way_en - io.data_read.bits.addr := xact.addr - io.data_write.valid := Bool(false) - io.data_write.bits.id := UInt(trackerId) - io.data_write.bits.way_en := xact_internal.way_en - io.data_write.bits.addr := xact.addr - io.data_write.bits.wmask := SInt(-1) - io.data_write.bits.data := xact.data - io.meta_read.valid := Bool(false) - io.meta_read.bits.id := UInt(trackerId) - io.meta_read.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta_read.bits.tag := xact.addr >> UInt(untagBits) - io.meta_write.valid := Bool(false) - io.meta_write.bits.id := UInt(trackerId) - io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta_write.bits.way_en := xact_internal.way_en - io.meta_write.bits.data := xact_internal.meta - - when(io.meta_resp.valid && io.meta_resp.bits.id === UInt(trackerId)) { xact_internal := io.meta_resp.bits } - - switch (state) { - is(s_idle) { - io.inner.acquire.ready := Bool(true) - val needs_outer_write = co.messageHasData(c_acq.payload) - val needs_outer_read = co.requiresOuterRead(c_acq.payload.a_type) - when( io.inner.acquire.valid ) { - xact := c_acq.payload - init_client_id := c_acq.header.src - probe_flags := probe_initial_flags - if(nClients > 1) { - release_count := PopCount(probe_initial_flags) - state := Mux(probe_initial_flags.orR, s_probe, - Mux(needs_outer_write, s_mem_write, - Mux(needs_outer_read, s_mem_read, s_make_grant))) - } else state := Mux(needs_outer_write, s_mem_write, - Mux(needs_outer_read, s_mem_read, s_make_grant)) - } - } - // s_read_meta - // xact_internal := resp - is(s_probe) { - // Generate probes - io.inner.probe.valid := probe_flags.orR - when(io.inner.probe.ready) { - probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) - } - - // Handle releases, which may have data to be written back - when(io.inner.release.valid) { - //xact_internal.meta.coh := tl.co.masterMetadataOnRelease( - // io.inner.release.bits.payload, - // xact_internal.meta.coh, - // init_client_id) - when(co.messageHasData(c_rel.payload)) { - io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_write_rel - when(io.outer.acquire.ready) { - io.inner.release.ready := Bool(true) - if(nClients > 1) release_count := release_count - UInt(1) - when(release_count === UInt(1)) { - state := Mux(pending_outer_write, s_mem_write, - Mux(pending_outer_read, s_mem_read, s_make_grant)) - } - } - } .otherwise { - io.inner.release.ready := Bool(true) - if(nClients > 1) release_count := release_count - UInt(1) - when(release_count === UInt(1)) { - state := Mux(pending_outer_write, s_mem_write, - Mux(pending_outer_read, s_mem_read, s_make_grant)) - } - } - } - } - is(s_mem_read) { - io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_read - when(io.outer.acquire.ready) { - state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) - } - } - is(s_mem_write) { - io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_write_acq - when(io.outer.acquire.ready) { - state := Mux(pending_outer_read, s_mem_read, s_make_grant) - } - } - is(s_make_grant) { - io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { - state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) - } - } - is(s_busy) { // Nothing left to do but wait for transaction to complete - when(io.outer.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { - io.inner.grant.valid := Bool(true) - } - when(io.inner.finish.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { - state := s_idle - } - } - } -} diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala deleted file mode 100644 index 35553702..00000000 --- a/uncore/src/main/scala/llc.scala +++ /dev/null @@ -1,554 +0,0 @@ -// See LICENSE for license details. - -package uncore -import Chisel._ - -class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UInt], noMask: Boolean = false)(gen: => T) extends Module -{ - class Inputs extends Bundle { - val addr = UInt(INPUT, log2Up(n)) - val rw = Bool(INPUT) - val wdata = gen.asInput - val wmask = gen.asInput - override def clone = new Inputs().asInstanceOf[this.type] - } - val io = new Bundle { - val in = Valid(new Inputs).flip - val rdata = gen.asOutput - } - val data = gen - val colMux = if (2*data.getWidth <= leaf.data.getWidth && n > leaf.n) 1 << math.floor(math.log(leaf.data.getWidth/data.getWidth)/math.log(2)).toInt else 1 - val nWide = if (data.getWidth > leaf.data.getWidth) 1+(data.getWidth-1)/leaf.data.getWidth else 1 - val nDeep = if (n > colMux*leaf.n) 1+(n-1)/(colMux*leaf.n) else 1 - if (nDeep > 1 || colMux > 1) - require(isPow2(n) && isPow2(leaf.n)) - - val rdataDeep = Vec.fill(nDeep){Bits()} - val rdataSel = Vec.fill(nDeep){Bool()} - for (i <- 0 until nDeep) { - val in = Pipe(io.in.valid && (if (nDeep == 1) Bool(true) else UInt(i) === io.in.bits.addr(log2Up(n)-1, log2Up(n/nDeep))), io.in.bits, preLatency) - val idx = in.bits.addr(log2Up(n/nDeep/colMux)-1, 0) - val wdata = in.bits.wdata.toBits - val wmask = in.bits.wmask.toBits - val ren = in.valid && !in.bits.rw - val reg_ren = Reg(next=ren) - val rdata = Vec.fill(nWide){Bits()} - - val r = Pipe(ren, in.bits.addr, postLatency) - - for (j <- 0 until nWide) { - val mem = leaf.clone - var dout: Bits = null - val ridx = if (postLatency > 0) Reg(Bits()) else null - - var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.getWidth*(j+1))-1, leaf.data.getWidth*j)) - if (colMux > 1) - wmask0 = wmask0 & FillInterleaved(gen.getWidth, UIntToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) - val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.getWidth*(j+1))-1, leaf.data.getWidth*j)) - when (in.valid) { - when (in.bits.rw) { - if (noMask) - mem.write(idx, wdata0) - else - mem.write(idx, wdata0, wmask0) - } - .otherwise { if (postLatency > 0) ridx := idx } - } - - if (postLatency == 0) { - dout = mem(idx) - } else if (postLatency == 1) { - dout = mem(ridx) - } else - dout = Pipe(reg_ren, mem(ridx), postLatency-1).bits - - rdata(j) := dout - } - val rdataWide = rdata.reduceLeft((x, y) => Cat(y, x)) - - var colMuxOut = rdataWide - if (colMux > 1) { - val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.getWidth*(k+1)-1, gen.getWidth*k))) - colMuxOut = colMuxIn(r.bits(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) - } - - rdataDeep(i) := colMuxOut - rdataSel(i) := r.valid - } - - io.rdata := Mux1H(rdataSel, rdataDeep) -} - -class LLCDataReq(ways: Int) extends MemReqCmd -{ - val way = UInt(width = log2Up(ways)) - val isWriteback = Bool() - override def clone = new LLCDataReq(ways).asInstanceOf[this.type] -} - -class LLCTagReq(ways: Int) extends HasMemAddr -{ - val way = UInt(width = log2Up(ways)) - override def clone = new LLCTagReq(ways).asInstanceOf[this.type] -} - -class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int) extends Module -{ - val io = new Bundle { - val cpu = Decoupled(new MemReqCmd).flip - val repl_way = UInt(INPUT, log2Up(ways)) - val repl_dirty = Bool(INPUT) - val repl_tag = UInt(INPUT, params(MIFAddrBits) - log2Up(sets)) - val data = Decoupled(new LLCDataReq(ways)) - val tag = Decoupled(new LLCTagReq(ways)) - val mem = new MemPipeIO - val mem_resp_set = UInt(OUTPUT, log2Up(sets)) - val mem_resp_way = UInt(OUTPUT, log2Up(ways)) - } - - class MSHR extends Bundle { - val addr = UInt(width = params(MIFAddrBits)) - val way = UInt(width = log2Up(ways)) - val tag = io.cpu.bits.tag.clone - val refilled = Bool() - val refillCount = UInt(width = log2Up(refill_cycles)) - val requested = Bool() - val old_dirty = Bool() - val old_tag = UInt(width = params(MIFAddrBits) - log2Up(sets)) - val wb_busy = Bool() - - override def clone = new MSHR().asInstanceOf[this.type] - } - - val valid = Vec.fill(outstanding){Reg(init=Bool(false))} - val validBits = valid.toBits - val freeId = PriorityEncoder(~validBits) - val mshr = Vec.fill(outstanding){Reg(new MSHR)} - when (io.cpu.valid && io.cpu.ready) { - valid(freeId) := Bool(true) - mshr(freeId).addr := io.cpu.bits.addr - mshr(freeId).tag := io.cpu.bits.tag - mshr(freeId).way := io.repl_way - mshr(freeId).old_dirty := io.repl_dirty - mshr(freeId).old_tag := io.repl_tag - mshr(freeId).wb_busy := Bool(false) - mshr(freeId).requested := Bool(false) - mshr(freeId).refillCount := UInt(0) - mshr(freeId).refilled := Bool(false) - } - - val requests = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && !mshr(i).old_dirty && !mshr(i).wb_busy && !mshr(i).requested):_*) - val request = requests.orR && io.data.ready // allow in-flight hits to drain - val requestId = PriorityEncoder(requests) - when (io.mem.req_cmd.valid && io.mem.req_cmd.ready) { mshr(requestId).requested := Bool(true) } - - val refillId = io.mem.resp.bits.tag(log2Up(outstanding)-1, 0) - val refillCount = mshr(refillId).refillCount - when (io.mem.resp.valid) { - mshr(refillId).refillCount := refillCount + UInt(1) - when (refillCount === UInt(refill_cycles-1)) { mshr(refillId).refilled := Bool(true) } - } - - val replays = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).refilled):_*) - val replay = replays.orR - val replayId = PriorityEncoder(replays) - when (replay && io.data.ready && io.tag.ready) { valid(replayId) := Bool(false) } - - val writebacks = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).old_dirty):_*) - val writeback = writebacks.orR - val writebackId = PriorityEncoder(writebacks) - when (writeback && io.data.ready && !replay) { - mshr(writebackId).old_dirty := Bool(false) - mshr(writebackId).wb_busy := Bool(true) - } - mshr.foreach(m => when (m.wb_busy && io.data.ready) { m.wb_busy := Bool(false) }) - - val conflicts = Cat(Bits(0), (0 until outstanding).map(i => valid(i) && io.cpu.bits.addr(log2Up(sets)-1, 0) === mshr(i).addr(log2Up(sets)-1, 0)):_*) - io.cpu.ready := !conflicts.orR && !validBits.andR - - io.data.valid := writeback - io.data.bits.rw := Bool(false) - io.data.bits.tag := mshr(replayId).tag - io.data.bits.isWriteback := Bool(true) - io.data.bits.addr := Cat(mshr(writebackId).old_tag, mshr(writebackId).addr(log2Up(sets)-1, 0)).toUInt - io.data.bits.way := mshr(writebackId).way - when (replay) { - io.data.valid := io.tag.ready - io.data.bits.isWriteback := Bool(false) - io.data.bits.addr := mshr(replayId).addr - io.data.bits.way := mshr(replayId).way - } - io.tag.valid := replay && io.data.ready - io.tag.bits.addr := io.data.bits.addr - io.tag.bits.way := io.data.bits.way - - io.mem.req_cmd.valid := request - io.mem.req_cmd.bits.rw := Bool(false) - io.mem.req_cmd.bits.addr := mshr(requestId).addr - io.mem.req_cmd.bits.tag := requestId - io.mem_resp_set := mshr(refillId).addr - io.mem_resp_way := mshr(refillId).way -} - -class LLCWriteback(requestors: Int, refill_cycles: Int) extends Module -{ - val io = new Bundle { - val req = Vec.fill(requestors){Decoupled(UInt(width = params(MIFAddrBits))).flip } - val data = Vec.fill(requestors){Decoupled(new MemData).flip } - val mem = new MemPipeIO - } - - val valid = Reg(init=Bool(false)) - val who = Reg(UInt()) - val addr = Reg(UInt()) - val cmd_sent = Reg(Bool()) - val data_sent = Reg(Bool()) - val count = Reg(init=UInt(0, log2Up(refill_cycles))) - - var anyReq = Bool(false) - for (i <- 0 until requestors) { - io.req(i).ready := !valid && !anyReq - io.data(i).ready := valid && who === UInt(i) && io.mem.req_data.ready - anyReq = anyReq || io.req(i).valid - } - - val nextWho = PriorityEncoder(io.req.map(_.valid)) - when (!valid && io.req.map(_.valid).reduceLeft(_||_)) { - valid := Bool(true) - cmd_sent := Bool(false) - data_sent := Bool(false) - who := nextWho - addr := io.req(nextWho).bits - } - - when (io.mem.req_data.valid && io.mem.req_data.ready) { - count := count + UInt(1) - when (count === UInt(refill_cycles-1)) { - data_sent := Bool(true) - when (cmd_sent) { valid := Bool(false) } - } - } - when (io.mem.req_cmd.valid && io.mem.req_cmd.ready) { cmd_sent := Bool(true) } - when (valid && cmd_sent && data_sent) { valid := Bool(false) } - - io.mem.req_cmd.valid := valid && !cmd_sent - io.mem.req_cmd.bits.addr := addr - io.mem.req_cmd.bits.rw := Bool(true) - - io.mem.req_data.valid := valid && !data_sent && io.data(who).valid - io.mem.req_data.bits := io.data(who).bits -} - -class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[UInt]) extends Module -{ - val io = new Bundle { - val req = Decoupled(new LLCDataReq(ways)).flip - val req_data = Decoupled(new MemData).flip - val writeback = Decoupled(UInt(width = params(MIFAddrBits))) - val writeback_data = Decoupled(new MemData) - val resp = Decoupled(new MemResp) - val mem_resp = Valid(new MemResp).flip - val mem_resp_set = UInt(INPUT, log2Up(sets)) - val mem_resp_way = UInt(INPUT, log2Up(ways)) - } - - val data = Module(new BigMem(sets*ways*refill_cycles, 1, latency-1, leaf, true)(Bits(width = params(MIFDataBits)))) - class QEntry extends MemResp { - val isWriteback = Bool() - override def clone = new QEntry().asInstanceOf[this.type] - } - val q = Module(new Queue(new QEntry, latency+2)) - val qReady = q.io.count <= UInt(q.entries-latency-1) - val valid = Reg(init=Bool(false)) - val req = Reg(io.req.bits.clone) - val count = Reg(init=UInt(0, log2Up(refill_cycles))) - val refillCount = Reg(init=UInt(0, log2Up(refill_cycles))) - - when (data.io.in.valid && !io.mem_resp.valid) { - count := count + UInt(1) - when (valid && count === UInt(refill_cycles-1)) { valid := Bool(false) } - } - when (io.req.valid && io.req.ready) { valid := Bool(true); req := io.req.bits } - when (io.mem_resp.valid) { refillCount := refillCount + UInt(1) } - - data.io.in.valid := io.req.valid && io.req.ready && Mux(io.req.bits.rw, io.req_data.valid, qReady) - data.io.in.bits.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUInt - data.io.in.bits.rw := io.req.bits.rw - data.io.in.bits.wdata := io.req_data.bits.data - when (valid) { - data.io.in.valid := Mux(req.rw, io.req_data.valid, qReady) - data.io.in.bits.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUInt - data.io.in.bits.rw := req.rw - } - when (io.mem_resp.valid) { - data.io.in.valid := Bool(true) - data.io.in.bits.addr := Cat(io.mem_resp_way, io.mem_resp_set, refillCount).toUInt - data.io.in.bits.rw := Bool(true) - data.io.in.bits.wdata := io.mem_resp.bits.data - } - - val tagPipe = Pipe(data.io.in.valid && !data.io.in.bits.rw, Mux(valid, req.tag, io.req.bits.tag), latency) - q.io.enq.valid := tagPipe.valid - q.io.enq.bits.tag := tagPipe.bits - q.io.enq.bits.isWriteback := Pipe(Mux(valid, req.isWriteback, io.req.bits.isWriteback), Bool(false), latency).valid - q.io.enq.bits.data := data.io.rdata - - io.req.ready := !valid && Mux(io.req.bits.isWriteback, io.writeback.ready, Bool(true)) - io.req_data.ready := !io.mem_resp.valid && Mux(valid, req.rw, io.req.valid && io.req.bits.rw) - - io.writeback.valid := io.req.valid && io.req.ready && io.req.bits.isWriteback - io.writeback.bits := io.req.bits.addr - - q.io.deq.ready := Mux(q.io.deq.bits.isWriteback, io.writeback_data.ready, io.resp.ready) - io.resp.valid := q.io.deq.valid && !q.io.deq.bits.isWriteback - io.resp.bits := q.io.deq.bits - io.writeback_data.valid := q.io.deq.valid && q.io.deq.bits.isWriteback - io.writeback_data.bits := q.io.deq.bits -} - -class MemReqArb(n: Int, refill_cycles: Int) extends Module -{ - val io = new Bundle { - val cpu = Vec.fill(n){new MemIO().flip} - val mem = new MemIO - } - - val lock = Reg(init=Bool(false)) - val locker = Reg(UInt()) - - val arb = Module(new RRArbiter(new MemReqCmd, n)) - val respWho = io.mem.resp.bits.tag(log2Up(n)-1,0) - val respTag = io.mem.resp.bits.tag >> UInt(log2Up(n)) - for (i <- 0 until n) { - val me = UInt(i, log2Up(n)) - arb.io.in(i).valid := io.cpu(i).req_cmd.valid - arb.io.in(i).bits := io.cpu(i).req_cmd.bits - arb.io.in(i).bits.tag := Cat(io.cpu(i).req_cmd.bits.tag, me) - io.cpu(i).req_cmd.ready := arb.io.in(i).ready - io.cpu(i).req_data.ready := Bool(false) - - val getLock = io.cpu(i).req_cmd.fire() && io.cpu(i).req_cmd.bits.rw && !lock - val haveLock = lock && locker === me - when (getLock) { - lock := Bool(true) - locker := UInt(i) - } - when (getLock || haveLock) { - io.cpu(i).req_data.ready := io.mem.req_data.ready - io.mem.req_data.valid := Bool(true) - io.mem.req_data.bits := io.cpu(i).req_data.bits - } - - io.cpu(i).resp.valid := io.mem.resp.valid && respWho === me - io.cpu(i).resp.bits := io.mem.resp.bits - io.cpu(i).resp.bits.tag := respTag - } - io.mem.resp.ready := io.cpu(respWho).resp.ready - - val unlock = Counter(io.mem.req_data.fire(), refill_cycles)._2 - when (unlock) { lock := Bool(false) } -} - -abstract class DRAMSideLLCLike extends Module { - val io = new Bundle { - val cpu = new MemIO().flip - val mem = new MemPipeIO - } -} - -// DRAMSideLLC has a known bug now. DO NOT USE. We are working on a brand new -// L2$. Stay stuned. -// -// Bug description: -// There's a race condition between the writeback unit (the module which -// sends the data out to the backside interface) and the data unit (the module -// which writes the data into the SRAM in the L2$). The "hit status" -// is saved in a register, however, is updated again when there's -// a transaction coming from the core without waiting until the writeback unit -// has sent out all its data to the outer memory system. This is why the -// problem manifests at a higher probability with the slow backup memory port. - -class DRAMSideLLC_HasKnownBug(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, tagLeaf: Mem[UInt], dataLeaf: Mem[UInt]) extends DRAMSideLLCLike -{ - val tagWidth = params(MIFAddrBits) - log2Up(sets) - val metaWidth = tagWidth + 2 // valid + dirty - - val memCmdArb = Module(new Arbiter(new MemReqCmd, 2)) - val dataArb = Module(new Arbiter(new LLCDataReq(ways), 2)) - val mshr = Module(new LLCMSHRFile(sets, ways, outstanding, refill_cycles)) - val tags = Module(new BigMem(sets, 0, 1, tagLeaf)(Bits(width = metaWidth*ways))) - val data = Module(new LLCData(4, sets, ways, refill_cycles, dataLeaf)) - val writeback = Module(new LLCWriteback(2, refill_cycles)) - - val initCount = Reg(init=UInt(0, log2Up(sets+1))) - val initialize = !initCount(log2Up(sets)) - when (initialize) { initCount := initCount + UInt(1) } - - val replay_s2 = Reg(init=Bool(false)) - val s2_valid = Reg(init=Bool(false)) - val s2 = Reg(new MemReqCmd) - val s3_rdy = Bool() - val replay_s2_rdy = Bool() - - val s1_valid = Reg(next = io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy, init = Bool(false)) - val s1 = Reg(new MemReqCmd) - when (io.cpu.req_cmd.fire()) { s1 := io.cpu.req_cmd.bits } - when (replay_s2 && replay_s2_rdy) { s1 := s2 } - - s2_valid := s1_valid - replay_s2 := s2_valid && !s3_rdy || replay_s2 && !replay_s2_rdy - val s2_tags = Vec.fill(ways){Reg(Bits(width = metaWidth))} - when (s1_valid) { - s2 := s1 - for (i <- 0 until ways) - s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) - } - val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.getWidth-1, s2.addr.getWidth-tagWidth) === t(tagWidth-1, 0)) - val s2_hit_way = OHToUInt(s2_hits) - val s2_hit = s2_hits.reduceLeft(_||_) - val s2_hit_dirty = s2_tags(s2_hit_way)(tagWidth+1) - val repl_way = LFSR16(s2_valid)(log2Up(ways)-1, 0) - val repl_tag = s2_tags(repl_way).toUInt - val setDirty = s2_valid && s2.rw && s2_hit && !s2_hit_dirty - - val tag_we = initialize || setDirty || mshr.io.tag.fire() - val tag_waddr = Mux(initialize, initCount, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)) - val tag_wdata = Cat(setDirty, !initialize, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.getWidth-1, mshr.io.tag.bits.addr.getWidth-tagWidth)) - val tag_wmask = Mux(initialize, SInt(-1, ways), UIntToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way))) - tags.io.in.valid := io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy || tag_we - tags.io.in.bits.addr := Mux(tag_we, tag_waddr, Mux(replay_s2, s2.addr, io.cpu.req_cmd.bits.addr)(log2Up(sets)-1,0)) - tags.io.in.bits.rw := tag_we - tags.io.in.bits.wdata := Fill(ways, tag_wdata) - tags.io.in.bits.wmask := FillInterleaved(metaWidth, tag_wmask) - - mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw - mshr.io.cpu.bits := s2 - mshr.io.repl_way := repl_way - mshr.io.repl_dirty := repl_tag(tagWidth+1, tagWidth).andR - mshr.io.repl_tag := repl_tag - mshr.io.mem.resp := io.mem.resp - mshr.io.tag.ready := !s1_valid && !s2_valid - - data.io.req <> dataArb.io.out - data.io.mem_resp := io.mem.resp - data.io.mem_resp_set := mshr.io.mem_resp_set - data.io.mem_resp_way := mshr.io.mem_resp_way - data.io.req_data.bits := io.cpu.req_data.bits - data.io.req_data.valid := io.cpu.req_data.valid - - writeback.io.req(0) <> data.io.writeback - writeback.io.data(0) <> data.io.writeback_data - writeback.io.req(1).valid := s2_valid && !s2_hit && s2.rw && mshr.io.cpu.ready - writeback.io.req(1).bits := s2.addr - writeback.io.data(1).valid := io.cpu.req_data.valid - writeback.io.data(1).bits := io.cpu.req_data.bits - - memCmdArb.io.in(0) <> mshr.io.mem.req_cmd - memCmdArb.io.in(1) <> writeback.io.mem.req_cmd - - dataArb.io.in(0) <> mshr.io.data - dataArb.io.in(1).valid := s2_valid && s2_hit && mshr.io.cpu.ready - dataArb.io.in(1).bits := s2 - dataArb.io.in(1).bits.way := s2_hit_way - dataArb.io.in(1).bits.isWriteback := Bool(false) - - s3_rdy := mshr.io.cpu.ready && Mux(s2_hit, dataArb.io.in(1).ready, !s2.rw || writeback.io.req(1).ready) - replay_s2_rdy := s3_rdy && !tag_we - - io.cpu.resp <> data.io.resp - io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready - io.mem.req_cmd <> memCmdArb.io.out - io.mem.req_data <> writeback.io.mem.req_data - io.cpu.req_cmd.ready := !(s1_valid || s2_valid || replay_s2 || tag_we || - io.cpu.req_cmd.bits.rw && io.cpu.req_data.ready) -} - -class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module -{ - val io = new QueueIO(data, entries) - require(isPow2(entries) && entries > 1) - - val do_flow = Bool() - val do_enq = io.enq.fire() && !do_flow - val do_deq = io.deq.fire() && !do_flow - - val maybe_full = Reg(init=Bool(false)) - val enq_ptr = Counter(do_enq, entries)._1 - val deq_ptr = Counter(do_deq, entries)._1 - when (do_enq != do_deq) { maybe_full := do_enq } - - val ptr_match = enq_ptr === deq_ptr - val empty = ptr_match && !maybe_full - val full = ptr_match && maybe_full - val atLeastTwo = full || enq_ptr - deq_ptr >= UInt(2) - do_flow := empty && io.deq.ready - - val ram = Mem(data, entries, seqRead = true) - val ram_addr = Reg(Bits()) - val ram_out_valid = Reg(Bool()) - ram_out_valid := Bool(false) - when (do_enq) { ram(enq_ptr) := io.enq.bits } - when (io.deq.ready && (atLeastTwo || !io.deq.valid && !empty)) { - ram_out_valid := Bool(true) - ram_addr := Mux(io.deq.valid, deq_ptr + UInt(1), deq_ptr) - } - - io.deq.valid := Mux(empty, io.enq.valid, ram_out_valid) - io.enq.ready := !full - io.deq.bits := Mux(empty, io.enq.bits, ram(ram_addr)) -} - -class HellaQueue[T <: Data](val entries: Int)(data: => T) extends Module -{ - val io = new QueueIO(data, entries) - - val fq = Module(new HellaFlowQueue(entries)(data)) - io.enq <> fq.io.enq - io.deq <> Queue(fq.io.deq, 1, pipe = true) -} - -object HellaQueue -{ - def apply[T <: Data](enq: DecoupledIO[T], entries: Int) = { - val q = Module((new HellaQueue(entries)) { enq.bits.clone }) - q.io.enq.valid := enq.valid // not using <> so that override is allowed - q.io.enq.bits := enq.bits - enq.ready := q.io.enq.ready - q.io.deq - } -} - -class DRAMSideLLCNull(numRequests: Int, refillCycles: Int) extends DRAMSideLLCLike { - val numEntries = numRequests * refillCycles - val size = log2Down(numEntries) + 1 - - val inc = Bool() - val dec = Bool() - val count = Reg(init=UInt(numEntries, size)) - val watermark = count >= UInt(refillCycles) - - when (inc && !dec) { - count := count + UInt(1) - } - when (!inc && dec) { - count := count - UInt(refillCycles) - } - when (inc && dec) { - count := count - UInt(refillCycles-1) - } - - val cmdq_mask = io.cpu.req_cmd.bits.rw || watermark - - io.mem.req_cmd.valid := io.cpu.req_cmd.valid && cmdq_mask - io.cpu.req_cmd.ready := io.mem.req_cmd.ready && cmdq_mask - io.mem.req_cmd.bits := io.cpu.req_cmd.bits - - io.mem.req_data <> io.cpu.req_data - - val resp_dataq = Module((new HellaQueue(numEntries)) { new MemResp }) - resp_dataq.io.enq <> io.mem.resp - io.cpu.resp <> resp_dataq.io.deq - - inc := resp_dataq.io.deq.fire() - dec := io.mem.req_cmd.fire() && !io.mem.req_cmd.bits.rw -} diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 6f42f010..b470df49 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -286,3 +286,112 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { } } +class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module +{ + val io = new QueueIO(data, entries) + require(isPow2(entries) && entries > 1) + + val do_flow = Bool() + val do_enq = io.enq.fire() && !do_flow + val do_deq = io.deq.fire() && !do_flow + + val maybe_full = Reg(init=Bool(false)) + val enq_ptr = Counter(do_enq, entries)._1 + val deq_ptr = Counter(do_deq, entries)._1 + when (do_enq != do_deq) { maybe_full := do_enq } + + val ptr_match = enq_ptr === deq_ptr + val empty = ptr_match && !maybe_full + val full = ptr_match && maybe_full + val atLeastTwo = full || enq_ptr - deq_ptr >= UInt(2) + do_flow := empty && io.deq.ready + + val ram = Mem(data, entries, seqRead = true) + val ram_addr = Reg(Bits()) + val ram_out_valid = Reg(Bool()) + ram_out_valid := Bool(false) + when (do_enq) { ram(enq_ptr) := io.enq.bits } + when (io.deq.ready && (atLeastTwo || !io.deq.valid && !empty)) { + ram_out_valid := Bool(true) + ram_addr := Mux(io.deq.valid, deq_ptr + UInt(1), deq_ptr) + } + + io.deq.valid := Mux(empty, io.enq.valid, ram_out_valid) + io.enq.ready := !full + io.deq.bits := Mux(empty, io.enq.bits, ram(ram_addr)) +} + +class HellaQueue[T <: Data](val entries: Int)(data: => T) extends Module +{ + val io = new QueueIO(data, entries) + + val fq = Module(new HellaFlowQueue(entries)(data)) + io.enq <> fq.io.enq + io.deq <> Queue(fq.io.deq, 1, pipe = true) +} + +object HellaQueue +{ + def apply[T <: Data](enq: DecoupledIO[T], entries: Int) = { + val q = Module((new HellaQueue(entries)) { enq.bits.clone }) + q.io.enq.valid := enq.valid // not using <> so that override is allowed + q.io.enq.bits := enq.bits + enq.ready := q.io.enq.ready + q.io.deq + } +} + +class MemPipeIOMemIOConverter(numRequests: Int, refillCycles: Int) extends Module { + val io = new Bundle { + val cpu = new MemIO().flip + val mem = new MemPipeIO + } + + val numEntries = numRequests * refillCycles + val size = log2Down(numEntries) + 1 + + val inc = Bool() + val dec = Bool() + val count = Reg(init=UInt(numEntries, size)) + val watermark = count >= UInt(refillCycles) + + when (inc && !dec) { + count := count + UInt(1) + } + when (!inc && dec) { + count := count - UInt(refillCycles) + } + when (inc && dec) { + count := count - UInt(refillCycles-1) + } + + val cmdq_mask = io.cpu.req_cmd.bits.rw || watermark + + io.mem.req_cmd.valid := io.cpu.req_cmd.valid && cmdq_mask + io.cpu.req_cmd.ready := io.mem.req_cmd.ready && cmdq_mask + io.mem.req_cmd.bits := io.cpu.req_cmd.bits + + io.mem.req_data <> io.cpu.req_data + + val resp_dataq = Module((new HellaQueue(numEntries)) { new MemResp }) + resp_dataq.io.enq <> io.mem.resp + io.cpu.resp <> resp_dataq.io.deq + + inc := resp_dataq.io.deq.fire() + dec := io.mem.req_cmd.fire() && !io.mem.req_cmd.bits.rw +} + +class MemPipeIOUncachedTileLinkIOConverter(outstanding: Int, refillCycles: Int) extends Module { + val io = new Bundle { + val uncached = new UncachedTileLinkIO().flip + val mem = new MemPipeIO + } + + val a = Module(new MemIOUncachedTileLinkIOConverter(2)) + val b = Module(new MemPipeIOMemIOConverter(outstanding, refillCycles)) + a.io.uncached <> io.uncached + b.io.cpu.req_cmd <> Queue(a.io.mem.req_cmd, 2) + b.io.cpu.req_data <> Queue(a.io.mem.req_data, refillCycles) + a.io.mem.resp <> b.io.cpu.resp + b.io.mem <> io.mem +} From d735f641102afb7805401dbc32c7e36a0b265904 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 2 Oct 2014 16:47:35 -0700 Subject: [PATCH 251/374] Parameter API update --- uncore/src/main/scala/tilelink.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 8bb98f0a..51864fe1 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -156,13 +156,13 @@ class Grant extends MasterSourcedMessage class Finish extends ClientSourcedMessage with HasMasterTransactionId -class UncachedTileLinkIO(p: Option[Parameters] = None) extends Bundle()(p) { +class UncachedTileLinkIO extends Bundle { val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) } -class TileLinkIO(p: Option[Parameters] = None) extends UncachedTileLinkIO(p) { +class TileLinkIO extends UncachedTileLinkIO { val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip val release = new DecoupledIO(new LogicalNetworkIO(new Release)) } From dc1a61264df319c05ac4c1d19eadabce9adfb7b6 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 30 Sep 2014 14:48:02 -0700 Subject: [PATCH 252/374] initial version, acts like old hub --- uncore/src/main/scala/cache.scala | 499 ++++++++++++++++++++++++++++++ 1 file changed, 499 insertions(+) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 26157a75..ca6b5c3f 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -96,3 +96,502 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { io.read.ready := !rst && !io.write.valid // so really this could be a 6T RAM io.write.ready := !rst } + +abstract trait L2HellaCacheParameters extends CacheParameters + with CoherenceAgentParameters + +abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters +abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters + +trait HasL2Id extends Bundle with CoherenceAgentParameters { + val id = UInt(width = log2Up(nTransactors)) +} + +trait HasL2InternalRequestState extends L2HellaCacheBundle { + val tag_match = Bool() + val meta = new L2Metadata + val way_en = Bits(width = nWays) +} + +object L2Metadata { + def apply(tag: Bits, coh: MasterMetadata) = { + val meta = new L2Metadata + meta.tag := tag + meta.coh := coh + meta + } +} +class L2Metadata extends Metadata with L2HellaCacheParameters { + val coh = co.masterMetadataOnFlush.clone +} + +class L2MetaReadReq extends MetaReadReq with HasL2Id { + val tag = Bits(width = tagBits) +} + +class L2MetaWriteReq extends MetaWriteReq[L2Metadata](new L2Metadata) + with HasL2Id + +class L2MetaResp extends L2HellaCacheBundle + with HasL2Id + with HasL2InternalRequestState + +class L2MetadataArray extends L2HellaCacheModule { + val io = new Bundle { + val read = Decoupled(new L2MetaReadReq).flip + val write = Decoupled(new L2MetaWriteReq).flip + val resp = Valid(new L2MetaResp) + } + + val meta = Module(new MetadataArray(() => L2Metadata(UInt(0), co.masterMetadataOnFlush))) + meta.io.read <> io.read + meta.io.write <> io.write + + val s1_clk_en = Reg(next = io.read.fire()) + val s1_tag = RegEnable(io.read.bits.tag, io.read.valid) + val s1_id = RegEnable(io.read.bits.id, io.read.valid) + def wayMap[T <: Data](f: Int => T) = Vec((0 until nWays).map(f)) + val s1_tag_eq_way = wayMap((w: Int) => meta.io.resp(w).tag === s1_tag) + val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && co.isValid(meta.io.resp(w).coh)).toBits + val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) + val s2_tag_match = s2_tag_match_way.orR + val s2_hit_coh = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).coh, s1_clk_en))) + //val s2_hit = s2_tag_match && tl.co.isHit(s2_req.cmd, s2_hit_state) && s2_hit_state === tl.co.newStateOnHit(s2_req.cmd, s2_hit_state) + + val replacer = params(Replacer)() + val s1_replaced_way_en = UIntToOH(replacer.way) + val s2_replaced_way_en = UIntToOH(RegEnable(replacer.way, s1_clk_en)) + val s2_repl_meta = Mux1H(s2_replaced_way_en, wayMap((w: Int) => + RegEnable(meta.io.resp(w), s1_clk_en && s1_replaced_way_en(w))).toSeq) + + io.resp.valid := Reg(next = s1_clk_en) + io.resp.bits.id := RegEnable(s1_id, s1_clk_en) + io.resp.bits.tag_match := s2_tag_match + io.resp.bits.meta := Mux(s2_tag_match, + L2Metadata(s2_repl_meta.tag, s2_hit_coh), + s2_repl_meta) + io.resp.bits.way_en := Mux(s2_tag_match, s2_tag_match_way, s2_replaced_way_en) +} + +class L2DataReadReq extends L2HellaCacheBundle with HasL2Id { + val way_en = Bits(width = nWays) + val addr = Bits(width = params(TLAddrBits)) +} + +class L2DataWriteReq extends L2DataReadReq { + val wmask = Bits(width = params(TLWriteMaskBits)) + val data = Bits(width = params(TLDataBits)) +} + +class L2DataResp extends Bundle with HasL2Id { + val data = Bits(width = params(TLDataBits)) +} + +class L2DataArray extends L2HellaCacheModule { + val io = new Bundle { + val read = Decoupled(new L2DataReadReq).flip + val write = Decoupled(new L2DataWriteReq).flip + val resp = Valid(new L2DataResp) + } + + val waddr = io.write.bits.addr + val raddr = io.read.bits.addr + val wmask = FillInterleaved(wordBits, io.write.bits.wmask) + val resp = (0 until nWays).map { w => + val array = Mem(Bits(width=params(RowBits)), nSets*refillCycles, seqRead = true) + when (io.write.bits.way_en(w) && io.write.valid) { + array.write(waddr, io.write.bits.data, wmask) + } + array(RegEnable(raddr, io.read.bits.way_en(w) && io.read.valid)) + } + io.resp.valid := ShiftRegister(io.read.valid, 2) + io.resp.bits.id := ShiftRegister(io.read.bits.id, 2) + io.resp.bits.data := Mux1H(ShiftRegister(io.read.bits.way_en, 2), resp) + + io.read.ready := Bool(true) + io.write.ready := Bool(true) +} + +class L2HellaCache(bankId: Int, innerId: String, outerId: String) extends + CoherenceAgent(innerId, outerId) with L2HellaCacheParameters { + + require(isPow2(nSets)) + require(isPow2(nWays)) + require(refillCycles == 1) + + val tshrfile = Module(new TSHRFile(bankId, innerId, outerId)) + val meta = Module(new L2MetadataArray) + val data = Module(new L2DataArray) + + tshrfile.io.inner <> io.inner + tshrfile.io.meta_read <> meta.io.read + tshrfile.io.meta_write <> meta.io.write + tshrfile.io.meta_resp <> meta.io.resp + tshrfile.io.data_read <> data.io.read + tshrfile.io.data_write <> data.io.write + tshrfile.io.data_resp <> data.io.resp + io.outer <> tshrfile.io.outer + io.incoherent <> tshrfile.io.incoherent +} + + +class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCacheModule { + val io = new Bundle { + val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip + val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) + val incoherent = Vec.fill(nClients){Bool()}.asInput + val meta_read = Decoupled(new L2MetaReadReq) + val meta_write = Decoupled(new L2MetaWriteReq) + val meta_resp = Valid(new L2MetaResp).flip + val data_read = Decoupled(new L2DataReadReq) + val data_write = Decoupled(new L2DataWriteReq) + val data_resp = Valid(new L2DataResp).flip + } + + // Wiring helper funcs + def doOutputArbitration[T <: Data](out: DecoupledIO[T], ins: Seq[DecoupledIO[T]]) { + val arb = Module(new RRArbiter(out.bits.clone, ins.size)) + out <> arb.io.out + arb.io.in zip ins map { case (a, in) => a <> in } + } + + def doInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { + outs.map(_.bits := in.bits) + outs.zipWithIndex.map { case (o, i) => o.valid := UInt(i) === in.bits.id } + } + + // Create TSHRs for outstanding transactions + val trackerList = (0 until nReleaseTransactors).map { id => + Module(new L2VoluntaryReleaseTracker(id, bankId, innerId, outerId)) + } ++ (nReleaseTransactors until nTransactors).map { id => + Module(new L2AcquireTracker(id, bankId, innerId, outerId)) + } + + // Propagate incoherence flags + trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) + + // Handle acquire transaction initiation + val acquire = io.inner.acquire + val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) + val block_acquires = any_acquire_conflict + + val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) + for( i <- 0 until trackerList.size ) { + val t = trackerList(i).io.inner + alloc_arb.io.in(i).valid := t.acquire.ready + t.acquire.bits := acquire.bits + t.acquire.valid := alloc_arb.io.in(i).ready + } + acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && !block_acquires + alloc_arb.io.out.ready := acquire.valid && !block_acquires + + // Handle probe requests + doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) + + // Handle releases, which might be voluntary and might have data + val release = io.inner.release + val voluntary = co.isVoluntary(release.bits.payload) + val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) + val block_releases = Bool(false) + val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b} + //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response + val release_idx = Mux(voluntary, UInt(0), release.bits.payload.master_xact_id) + for( i <- 0 until trackerList.size ) { + val t = trackerList(i).io.inner + t.release.bits := release.bits + t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases + } + release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases + + // Reply to initial requestor + doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) + + // Free finished transactions + val ack = io.inner.finish + trackerList.map(_.io.inner.finish.valid := ack.valid) + trackerList.map(_.io.inner.finish.bits := ack.bits) + ack.ready := Bool(true) + + // Arbitrate for the outer memory port + val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), + {case TLId => outerId}) + outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } + io.outer <> outer_arb.io.out + + // Local memory + doOutputArbitration(io.meta_read, trackerList.map(_.io.meta_read)) + doOutputArbitration(io.meta_write, trackerList.map(_.io.meta_write)) + doOutputArbitration(io.data_read, trackerList.map(_.io.data_read)) + doOutputArbitration(io.data_write, trackerList.map(_.io.data_write)) + doInputRouting(io.meta_resp, trackerList.map(_.io.meta_resp)) + doInputRouting(io.data_resp, trackerList.map(_.io.data_resp)) + +} + + +abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCacheModule { + val io = new Bundle { + val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip + val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) + val tile_incoherent = Bits(INPUT, nClients) + val has_acquire_conflict = Bool(OUTPUT) + val has_release_conflict = Bool(OUTPUT) + val meta_read = Decoupled(new L2MetaReadReq) + val meta_write = Decoupled(new L2MetaWriteReq) + val meta_resp = Valid(new L2MetaResp).flip + val data_read = Decoupled(new L2DataReadReq) + val data_write = Decoupled(new L2DataWriteReq) + val data_resp = Valid(new L2DataResp).flip + } + + val c_acq = io.inner.acquire.bits + val c_rel = io.inner.release.bits + val c_gnt = io.inner.grant.bits + val c_ack = io.inner.finish.bits + val m_gnt = io.outer.grant.bits + +} + +class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { + val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) + val state = Reg(init=s_idle) + val xact = Reg{ new Release } + val xact_internal = Reg{ new L2MetaResp } + val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) + + io.has_acquire_conflict := Bool(false) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && + (state != s_idle) + + io.outer.grant.ready := Bool(false) + io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.header.src := UInt(bankId) + io.outer.acquire.bits.payload := Bundle(Acquire(co.getUncachedWriteAcquireType, + xact.addr, + UInt(trackerId), + xact.data), + { case TLId => outerId }) + io.inner.acquire.ready := Bool(false) + io.inner.probe.valid := Bool(false) + io.inner.release.ready := Bool(false) + io.inner.grant.valid := Bool(false) + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := init_client_id + io.inner.grant.bits.payload := Grant(co.getGrantType(xact, co.masterMetadataOnFlush),// TODO xact_internal.meta) + xact.client_xact_id, + UInt(trackerId)) + + io.data_read.valid := Bool(false) + io.data_write.valid := Bool(false) + io.data_write.bits.id := UInt(trackerId) + io.data_write.bits.way_en := xact_internal.way_en + io.data_write.bits.addr := xact.addr + io.data_write.bits.wmask := SInt(-1) + io.data_write.bits.data := xact.data + io.meta_read.valid := Bool(false) + io.meta_read.bits.id := UInt(trackerId) + io.meta_read.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta_read.bits.tag := xact.addr >> UInt(untagBits) + io.meta_write.valid := Bool(false) + io.meta_write.bits.id := UInt(trackerId) + io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta_write.bits.way_en := xact_internal.way_en + io.meta_write.bits.data := xact_internal.meta + + when(io.meta_resp.valid) { xact_internal := io.meta_resp.bits } + + switch (state) { + is(s_idle) { + io.inner.release.ready := Bool(true) + when( io.inner.release.valid ) { + xact := c_rel.payload + init_client_id := c_rel.header.src + state := s_mem + } + }/* + is(s_meta_read) { + when(io.meta_read.ready) state := s_meta_resp + } + is(s_meta_resp) { + when(io.meta_resp.valid) { + xact_internal.meta := tl.co.masterMetadataOnRelease(xact, xact_internal.meta, init_client_id)) + state := Mux(s_meta_write + Mux(co.messageHasData(xact), s_mem, s_ack) + }*/ + is(s_mem) { + io.outer.acquire.valid := Bool(true) + when(io.outer.acquire.ready) { state := s_ack } + } + is(s_ack) { + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { state := s_idle } + } + } +} + +class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { + val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) + val state = Reg(init=s_idle) + val xact = Reg{ new Acquire } + val xact_internal = Reg{ new L2MetaResp } + val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) + //TODO: Will need id reg for merged release xacts + + val release_count = if (nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(nClients))) + val probe_flags = Reg(init=Bits(0, width = nClients)) + val curr_p_id = PriorityEncoder(probe_flags) + + val pending_outer_write = co.messageHasData(xact) + val pending_outer_read = co.requiresOuterRead(xact.a_type) + val outer_write_acq = Bundle(Acquire(co.getUncachedWriteAcquireType, + xact.addr, UInt(trackerId), xact.data), + { case TLId => outerId }) + val outer_write_rel = Bundle(Acquire(co.getUncachedWriteAcquireType, + xact.addr, UInt(trackerId), c_rel.payload.data), + { case TLId => outerId }) + val outer_read = Bundle(Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)), + { case TLId => outerId }) + + val probe_initial_flags = Bits(width = nClients) + probe_initial_flags := Bits(0) + if (nClients > 1) { + // issue self-probes for uncached read xacts to facilitate I$ coherence + val probe_self = Bool(true) //co.needsSelfProbe(io.inner.acquire.bits.payload) + val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(nClients)-1,0))) + probe_initial_flags := ~(io.tile_incoherent | myflag) + } + + io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) + + io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.header.src := UInt(bankId) + io.outer.acquire.bits.payload := outer_read + io.outer.grant.ready := io.inner.grant.ready + + io.inner.probe.valid := Bool(false) + io.inner.probe.bits.header.src := UInt(bankId) + io.inner.probe.bits.header.dst := curr_p_id + io.inner.probe.bits.payload := Probe(co.getProbeType(xact, /*xact_internal.meta), TODO*/ + co.masterMetadataOnFlush), + xact.addr, + UInt(trackerId)) + + val grant_type = co.getGrantType(xact, co.masterMetadataOnFlush)// TODO xact_internal.meta) + io.inner.grant.valid := Bool(false) + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := init_client_id + io.inner.grant.bits.payload := Grant(grant_type, + xact.client_xact_id, + UInt(trackerId), + m_gnt.payload.data) + + io.inner.acquire.ready := Bool(false) + io.inner.release.ready := Bool(false) + + io.data_read.valid := Bool(false) + io.data_read.bits.id := UInt(trackerId) + io.data_read.bits.way_en := xact_internal.way_en + io.data_read.bits.addr := xact.addr + io.data_write.valid := Bool(false) + io.data_write.bits.id := UInt(trackerId) + io.data_write.bits.way_en := xact_internal.way_en + io.data_write.bits.addr := xact.addr + io.data_write.bits.wmask := SInt(-1) + io.data_write.bits.data := xact.data + io.meta_read.valid := Bool(false) + io.meta_read.bits.id := UInt(trackerId) + io.meta_read.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta_read.bits.tag := xact.addr >> UInt(untagBits) + io.meta_write.valid := Bool(false) + io.meta_write.bits.id := UInt(trackerId) + io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta_write.bits.way_en := xact_internal.way_en + io.meta_write.bits.data := xact_internal.meta + + when(io.meta_resp.valid && io.meta_resp.bits.id === UInt(trackerId)) { xact_internal := io.meta_resp.bits } + + switch (state) { + is(s_idle) { + io.inner.acquire.ready := Bool(true) + val needs_outer_write = co.messageHasData(c_acq.payload) + val needs_outer_read = co.requiresOuterRead(c_acq.payload.a_type) + when( io.inner.acquire.valid ) { + xact := c_acq.payload + init_client_id := c_acq.header.src + probe_flags := probe_initial_flags + if(nClients > 1) { + release_count := PopCount(probe_initial_flags) + state := Mux(probe_initial_flags.orR, s_probe, + Mux(needs_outer_write, s_mem_write, + Mux(needs_outer_read, s_mem_read, s_make_grant))) + } else state := Mux(needs_outer_write, s_mem_write, + Mux(needs_outer_read, s_mem_read, s_make_grant)) + } + } + // s_read_meta + // xact_internal := resp + is(s_probe) { + // Generate probes + io.inner.probe.valid := probe_flags.orR + when(io.inner.probe.ready) { + probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) + } + + // Handle releases, which may have data to be written back + when(io.inner.release.valid) { + //xact_internal.meta.coh := tl.co.masterMetadataOnRelease( + // io.inner.release.bits.payload, + // xact_internal.meta.coh, + // init_client_id) + when(co.messageHasData(c_rel.payload)) { + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_write_rel + when(io.outer.acquire.ready) { + io.inner.release.ready := Bool(true) + if(nClients > 1) release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } + } + } .otherwise { + io.inner.release.ready := Bool(true) + if(nClients > 1) release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } + } + } + } + is(s_mem_read) { + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_read + when(io.outer.acquire.ready) { + state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) + } + } + is(s_mem_write) { + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_write_acq + when(io.outer.acquire.ready) { + state := Mux(pending_outer_read, s_mem_read, s_make_grant) + } + } + is(s_make_grant) { + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { + state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) + } + } + is(s_busy) { // Nothing left to do but wait for transaction to complete + when(io.outer.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { + io.inner.grant.valid := Bool(true) + } + when(io.inner.finish.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { + state := s_idle + } + } + } +} From 394eb38a963588427fe900c7bffa369dc494ab88 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 3 Oct 2014 01:06:33 -0700 Subject: [PATCH 253/374] temp; converted voluntary wb tracker --- uncore/src/main/scala/cache.scala | 56 ++++++++++++++------------- uncore/src/main/scala/coherence.scala | 4 +- 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index ca6b5c3f..b4e210b0 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -122,7 +122,7 @@ object L2Metadata { } } class L2Metadata extends Metadata with L2HellaCacheParameters { - val coh = co.masterMetadataOnFlush.clone + val coh = new MasterMetadata()(co) //co.masterMetadataOnFlush.clone } class L2MetaReadReq extends MetaReadReq with HasL2Id { @@ -306,11 +306,14 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac // Reply to initial requestor doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) - // Free finished transactions - val ack = io.inner.finish - trackerList.map(_.io.inner.finish.valid := ack.valid) - trackerList.map(_.io.inner.finish.bits := ack.bits) - ack.ready := Bool(true) + // Free finished transactions on ack + val finish = io.inner.finish + val finish_idx = finish.bits.payload.master_xact_id + trackerList.zipWithIndex.map { case (t, i) => + t.io.inner.finish.valid := finish.valid && finish_idx === UInt(i) + } + trackerList.map(_.io.inner.finish.bits := finish.bits) + finish.ready := Vec(trackerList.map(_.io.inner.finish.ready)).read(finish_idx) // Arbitrate for the outer memory port val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), @@ -353,7 +356,7 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa } class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) + val s_idle :: s_meta_read :: s_meta_resp :: s_meta_write :: s_data_write :: s_ack :: s_busy :: Nil = Enum(UInt(), 6) val state = Reg(init=s_idle) val xact = Reg{ new Release } val xact_internal = Reg{ new L2MetaResp } @@ -365,19 +368,14 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.outer.grant.ready := Bool(false) io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits.header.src := UInt(bankId) - io.outer.acquire.bits.payload := Bundle(Acquire(co.getUncachedWriteAcquireType, - xact.addr, - UInt(trackerId), - xact.data), - { case TLId => outerId }) + io.inner.acquire.ready := Bool(false) io.inner.probe.valid := Bool(false) io.inner.release.ready := Bool(false) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id - io.inner.grant.bits.payload := Grant(co.getGrantType(xact, co.masterMetadataOnFlush),// TODO xact_internal.meta) + io.inner.grant.bits.payload := Grant(co.getGrantType(xact, xact_internal.meta.coh), xact.client_xact_id, UInt(trackerId)) @@ -397,8 +395,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) io.meta_write.bits.way_en := xact_internal.way_en io.meta_write.bits.data := xact_internal.meta - - when(io.meta_resp.valid) { xact_internal := io.meta_resp.bits } + io.meta_resp.valid := Bool(true) switch (state) { is(s_idle) { @@ -406,25 +403,32 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou when( io.inner.release.valid ) { xact := c_rel.payload init_client_id := c_rel.header.src - state := s_mem + state := s_meta_read } - }/* + } is(s_meta_read) { - when(io.meta_read.ready) state := s_meta_resp + io.meta_read.valid := Bool(true) + when(io.meta_read.ready) { state := s_meta_resp } } is(s_meta_resp) { when(io.meta_resp.valid) { - xact_internal.meta := tl.co.masterMetadataOnRelease(xact, xact_internal.meta, init_client_id)) - state := Mux(s_meta_write - Mux(co.messageHasData(xact), s_mem, s_ack) - }*/ - is(s_mem) { - io.outer.acquire.valid := Bool(true) + xact_internal := io.meta_resp.bits + state := s_meta_write + } + } + is(s_meta_write) { + io.meta_write.valid := Bool(true) when(io.outer.acquire.ready) { state := s_ack } } is(s_ack) { io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { state := s_idle } + when(io.inner.grant.ready) { + state := Mux(co.requiresAckForGrant(io.inner.grant.bits.payload.g_type), + s_busy, s_idle) + } + } + is(s_busy) { + when(io.inner.finish.valid) { state := s_idle } } } } diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 7dbff4aa..778a04f6 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -62,10 +62,10 @@ abstract class DirectoryRepresentation extends Bundle { } class NullRepresentation extends DirectoryRepresentation { - val internal = UInt(0) + val internal = UInt() def pop(id: UInt) = this def push(id: UInt) = this - def flush(dummy: Int = 0) = this + def flush(dummy: Int = 0) = { internal := UInt(0); this } def none(dummy: Int = 0) = Bool(false) def one(dummy: Int = 0) = Bool(false) def count(dummy: Int = 0) = UInt(0) From 86bdbd65358e253f7742927f74da69c7e9af494e Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 7 Oct 2014 22:33:10 -0700 Subject: [PATCH 254/374] new tshrs, compiles but does not elaborate --- uncore/src/main/scala/cache.scala | 275 +++++++++++++++++--------- uncore/src/main/scala/coherence.scala | 204 ++++++++++--------- uncore/src/main/scala/tilelink.scala | 4 + 3 files changed, 293 insertions(+), 190 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index b4e210b0..9cc99e4e 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -353,10 +353,14 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa val c_ack = io.inner.finish.bits val m_gnt = io.outer.grant.bits + def mergeData(acq: Acquire, data: UInt): UInt = { + //TODO wite mask + Mux(co.messageHasData(acq), acq.data, data) + } } class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_meta_read :: s_meta_resp :: s_meta_write :: s_data_write :: s_ack :: s_busy :: Nil = Enum(UInt(), 6) + val s_idle :: s_meta_read :: s_meta_resp :: s_meta_write :: s_data_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 6) val state = Reg(init=s_idle) val xact = Reg{ new Release } val xact_internal = Reg{ new L2MetaResp } @@ -412,18 +416,26 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou } is(s_meta_resp) { when(io.meta_resp.valid) { - xact_internal := io.meta_resp.bits - state := s_meta_write + xact_internal := co.masterMetadataOnRelease(xact, + io.meta_resp.bits.meta.coh, + init_client_id) + state := Mux(io.meta_resp.bits.tag_match, + Mux(co.messageHasData(xact), s_data_write, s_meta_write), + s_grant) } } + is(s_data_write) { + io.data_write.valid := Bool(true) + when(io.data_write.ready) { state := s_meta_write } + } is(s_meta_write) { io.meta_write.valid := Bool(true) - when(io.outer.acquire.ready) { state := s_ack } + when(io.meta_write.ready) { state := s_grant } } - is(s_ack) { + is(s_grant) { io.inner.grant.valid := Bool(true) when(io.inner.grant.ready) { - state := Mux(co.requiresAckForGrant(io.inner.grant.bits.payload.g_type), + state := Mux(co.requiresAckForGrant(c_gnt.payload.g_type), s_busy, s_idle) } } @@ -434,61 +446,71 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou } class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) + val s_idle :: s_meta_read :: s_meta_resp :: s_probe :: s_data_read_wb :: s_data_resp_wb :: s_outer_write_wb :: s_outer_read :: s_outer_resp :: s_data_read_hit :: s_data_resp_hit :: s_data_write :: s_outer_write_acq :: s_meta_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 16) val state = Reg(init=s_idle) val xact = Reg{ new Acquire } val xact_internal = Reg{ new L2MetaResp } val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) //TODO: Will need id reg for merged release xacts - val release_count = if (nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(nClients))) - val probe_flags = Reg(init=Bits(0, width = nClients)) - val curr_p_id = PriorityEncoder(probe_flags) - val pending_outer_write = co.messageHasData(xact) - val pending_outer_read = co.requiresOuterRead(xact.a_type) + val release_count = Reg(init = UInt(0, width = log2Up(nClients))) + val pending_probes = Reg(init = co.dir()) + val curr_p_id = pending_probes.next() + + val is_uncached = co.messageIsUncached(xact) + val tag_match = xact_internal.tag_match + val needs_writeback = co.needsWriteback(xact_internal.meta.coh) + val is_hit = co.isHit(xact, xact_internal.meta.coh) + val needs_probes = co.requiresProbes(xact.a_type, xact_internal.meta.coh) + val c_rel_had_data = Reg{Bool()} + val c_rel_was_voluntary = Reg{Bool()} + val wb_buffer = Reg{xact.data.clone} + + io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && + (state != s_idle) //TODO: Also indexes + io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && + (state != s_idle) //TODO: Also indexes? + val outer_write_acq = Bundle(Acquire(co.getUncachedWriteAcquireType, - xact.addr, UInt(trackerId), xact.data), - { case TLId => outerId }) - val outer_write_rel = Bundle(Acquire(co.getUncachedWriteAcquireType, - xact.addr, UInt(trackerId), c_rel.payload.data), - { case TLId => outerId }) - val outer_read = Bundle(Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)), - { case TLId => outerId }) - - val probe_initial_flags = Bits(width = nClients) - probe_initial_flags := Bits(0) - if (nClients > 1) { - // issue self-probes for uncached read xacts to facilitate I$ coherence - val probe_self = Bool(true) //co.needsSelfProbe(io.inner.acquire.bits.payload) - val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(nClients)-1,0))) - probe_initial_flags := ~(io.tile_incoherent | myflag) - } - - io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) - + xact.addr, + UInt(trackerId), + xact.data), { case TLId => outerId }) + val outer_write_wb = Bundle(Acquire(co.getUncachedWriteAcquireType, + Cat(xact_internal.meta.tag, + xact.addr(untagBits-1,blockOffBits)), + UInt(trackerId), + wb_buffer), { case TLId => outerId }) + val outer_read = Bundle(Acquire(co.getUncachedReadAcquireType, + xact.addr, + UInt(trackerId)), { case TLId => outerId }) io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.payload := outer_read //default io.outer.acquire.bits.header.src := UInt(bankId) - io.outer.acquire.bits.payload := outer_read - io.outer.grant.ready := io.inner.grant.ready + io.outer.grant.ready := Bool(true) //grant.data -> xact.data + val inner_probe_cacq = Probe(co.getProbeType(xact, xact_internal.meta.coh), + xact.addr, + UInt(trackerId)) + val inner_probe_wb = Probe(co.getProbeTypeOnVoluntaryWriteback, + xact.addr, + UInt(trackerId)) + //TODO inner_probe_mprb io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact, /*xact_internal.meta), TODO*/ - co.masterMetadataOnFlush), - xact.addr, - UInt(trackerId)) + io.inner.probe.bits.payload := Mux(needs_writeback, + inner_probe_wb, + inner_probe_cacq) - val grant_type = co.getGrantType(xact, co.masterMetadataOnFlush)// TODO xact_internal.meta) + val grant_type = co.getGrantType(xact, xact_internal.meta.coh) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id io.inner.grant.bits.payload := Grant(grant_type, xact.client_xact_id, UInt(trackerId), - m_gnt.payload.data) + xact.data) io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) @@ -511,91 +533,154 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.meta_write.bits.id := UInt(trackerId) io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) io.meta_write.bits.way_en := xact_internal.way_en - io.meta_write.bits.data := xact_internal.meta - - when(io.meta_resp.valid && io.meta_resp.bits.id === UInt(trackerId)) { xact_internal := io.meta_resp.bits } + io.meta_write.bits.data.tag := xact.addr >> UInt(untagBits) + io.meta_write.bits.data.coh := xact_internal.meta.coh switch (state) { is(s_idle) { io.inner.acquire.ready := Bool(true) - val needs_outer_write = co.messageHasData(c_acq.payload) - val needs_outer_read = co.requiresOuterRead(c_acq.payload.a_type) when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src - probe_flags := probe_initial_flags - if(nClients > 1) { - release_count := PopCount(probe_initial_flags) - state := Mux(probe_initial_flags.orR, s_probe, - Mux(needs_outer_write, s_mem_write, - Mux(needs_outer_read, s_mem_read, s_make_grant))) - } else state := Mux(needs_outer_write, s_mem_write, - Mux(needs_outer_read, s_mem_read, s_make_grant)) + state := s_meta_read } } - // s_read_meta - // xact_internal := resp - is(s_probe) { - // Generate probes - io.inner.probe.valid := probe_flags.orR - when(io.inner.probe.ready) { - probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) + is(s_meta_read) { + io.meta_read.valid := Bool(true) + when(io.meta_read.ready) { state := s_meta_resp } + } + is(s_meta_resp) { + when(io.meta_resp.valid) { + val coh = io.meta_resp.bits.meta.coh + val _tag_match = io.meta_resp.bits.tag_match + val _needs_writeback = co.needsWriteback(coh) + val _is_hit = co.isHit(xact, coh) + val _needs_probes = co.requiresProbes(xact.a_type, coh) + xact_internal := io.meta_resp.bits + when(!_needs_writeback) { + xact_internal.meta.coh := co.masterMetadataOnFlush + } + when(_needs_probes) { + pending_probes := coh.sharers + release_count := coh.sharers.count() + c_rel_had_data := Bool(false) + c_rel_was_voluntary := Bool(false) + } + state := Mux(_tag_match, + Mux(_is_hit, + Mux(_needs_probes, s_probe, s_data_read_hit), + Mux(_needs_probes, s_probe, s_outer_read)), + Mux(_needs_writeback, + Mux(_needs_probes, s_probe, s_data_read_wb), + s_outer_read)) } + } + is(s_probe) { + val skip = io.tile_incoherent(curr_p_id) || + ((curr_p_id === init_client_id) && + !co.requiresSelfProbe(xact.a_type)) + io.inner.probe.valid := !(pending_probes.none() || skip) + when(io.inner.probe.ready || skip) { + pending_probes.pop(curr_p_id) + } + when(skip) { release_count := release_count - UInt(1) } - // Handle releases, which may have data to be written back + // Handle releases, which may have data being written back + io.inner.release.ready := Bool(true) when(io.inner.release.valid) { - //xact_internal.meta.coh := tl.co.masterMetadataOnRelease( - // io.inner.release.bits.payload, - // xact_internal.meta.coh, - // init_client_id) + xact_internal.meta.coh := co.masterMetadataOnRelease( + c_rel.payload, + xact_internal.meta.coh, + c_rel.header.src) when(co.messageHasData(c_rel.payload)) { - io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_write_rel - when(io.outer.acquire.ready) { - io.inner.release.ready := Bool(true) - if(nClients > 1) release_count := release_count - UInt(1) - when(release_count === UInt(1)) { - state := Mux(pending_outer_write, s_mem_write, - Mux(pending_outer_read, s_mem_read, s_make_grant)) - } + c_rel_had_data := Bool(true) + when(tag_match) { + xact.data := mergeData(xact, io.inner.release.bits.payload.data) + } .otherwise { + wb_buffer := io.inner.release.bits.payload.data } - } .otherwise { - io.inner.release.ready := Bool(true) - if(nClients > 1) release_count := release_count - UInt(1) + } + when(co.isVoluntary(c_rel.payload)) { + c_rel_was_voluntary := Bool(true) + } + when(!co.isVoluntary(c_rel.payload)) { + release_count := release_count - Mux(skip, UInt(2), UInt(1)) when(release_count === UInt(1)) { - state := Mux(pending_outer_write, s_mem_write, - Mux(pending_outer_read, s_mem_read, s_make_grant)) + state := Mux(tag_match, + Mux(is_hit, + Mux(c_rel_had_data, s_meta_write, s_data_read_hit), + s_outer_read), + Mux(c_rel_had_data, s_outer_write_wb, s_data_read_wb)) } } } } - is(s_mem_read) { + is(s_data_read_wb) { + io.data_read.valid := Bool(true) + when(io.data_read.ready) { + state := s_data_resp_wb + } + } + is(s_data_resp_wb) { + when(io.data_resp.valid) { + wb_buffer := io.data_resp.bits + state := s_outer_write_wb + } + } + is(s_outer_write_wb) { + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_write_wb + when(io.outer.acquire.ready) { + state := s_outer_read + } + } + is(s_outer_read) { io.outer.acquire.valid := Bool(true) io.outer.acquire.bits.payload := outer_read when(io.outer.acquire.ready) { - state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) + state := s_outer_resp } } - is(s_mem_write) { - io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_write_acq - when(io.outer.acquire.ready) { - state := Mux(pending_outer_read, s_mem_read, s_make_grant) + is(s_outer_resp) { + io.outer.grant.ready := Bool(true) + when(io.outer.grant.valid) { + xact.data := mergeData(xact, io.outer.grant.bits.payload.data) + //TODO: set pending client state in xact_internal.meta.coh + state := Mux(co.messageHasData(io.outer.grant.bits.payload), + s_data_write, s_data_read_hit) } } - is(s_make_grant) { + is(s_data_read_hit) { + io.data_read.valid := Bool(true) + when(io.data_read.ready) { + state := s_data_resp_hit + } + } + is(s_data_resp_hit) { + when(io.data_resp.valid) { + xact.data := mergeData(xact, io.data_resp.bits.data) + state := s_meta_write + } + } + is(s_data_write) { + io.data_write.valid := Bool(true) + when(io.data_write.ready) { + state := s_meta_write + } + } + is(s_meta_write) { + io.meta_write.valid := Bool(true) + when(io.meta_write.ready) { state := s_grant } + } + is(s_grant) { io.inner.grant.valid := Bool(true) when(io.inner.grant.ready) { - state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) + state := Mux(co.requiresAckForGrant(c_gnt.payload.g_type), + s_busy, s_idle) } } - is(s_busy) { // Nothing left to do but wait for transaction to complete - when(io.outer.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { - io.inner.grant.valid := Bool(true) - } - when(io.inner.finish.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { - state := s_idle - } + is(s_busy) { + when(io.inner.finish.valid) { state := s_idle } } } } diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 778a04f6..5dd87b35 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -51,7 +51,6 @@ class MixedMetadata(inner: CoherencePolicy, outer: CoherencePolicy) extends Cohe */ abstract class DirectoryRepresentation extends Bundle { - val internal: UInt def pop(id: UInt): DirectoryRepresentation def push(id: UInt): DirectoryRepresentation def flush(dummy: Int = 0): DirectoryRepresentation @@ -106,6 +105,7 @@ abstract class CoherencePolicy(val dir: () => DirectoryRepresentation) { def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool def needsWriteback(m: ClientMetadata): Bool + def needsWriteback(m: MasterMetadata): Bool def clientMetadataOnHit(cmd: UInt, m: ClientMetadata): ClientMetadata def clientMetadataOnCacheControl(cmd: UInt): ClientMetadata @@ -114,33 +114,33 @@ abstract class CoherencePolicy(val dir: () => DirectoryRepresentation) { def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata): ClientMetadata def masterMetadataOnFlush: MasterMetadata def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt): MasterMetadata + def masterMetadataOnGrant(outgoing: Grant, m: MasterMetadata, dst: UInt): MasterMetadata def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt def getProbeType(a: Acquire, m: MasterMetadata): UInt + def getProbeTypeOnVoluntaryWriteback: UInt def getReleaseTypeOnCacheControl(cmd: UInt): UInt def getReleaseTypeOnVoluntaryWriteback(): UInt def getReleaseTypeOnProbe(p: Probe, m: ClientMetadata): UInt def getGrantType(a: Acquire, m: MasterMetadata): UInt def getGrantType(r: Release, m: MasterMetadata): UInt - //def getGrantType(a: Acquire) = getGrantType(a, new NullRepresentation) // TODO - //def getGrantType(r: Release) = getGrantType(r, new NullRepresentation) def messageHasData (rel: SourcedMessage): Bool def messageUpdatesDataArray (reply: Grant): Bool def messageIsUncached(acq: Acquire): Bool - - def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool def isVoluntary(rel: Release): Bool def isVoluntary(gnt: Grant): Bool - def requiresOuterRead(a_type: UInt, m: MasterMetadata): Bool - def requiresOuterWrite(a_type: UInt, m: MasterMetadata): Bool + def requiresOuterRead(a_type: UInt): Bool def requiresOuterWrite(a_type: UInt): Bool def requiresSelfProbe(a_type: UInt): Bool + def requiresProbes(a_type: UInt, m: MasterMetadata): Bool def requiresAckForGrant(g_type: UInt): Bool def requiresAckForRelease(r_type: UInt): Bool def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool + + def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool } trait UncachedTransactions { @@ -191,6 +191,7 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit def needsWriteback (m: ClientMetadata): Bool = { needsTransactionOnCacheControl(M_INV, m) } + def needsWriteback(m: MasterMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = m def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( @@ -214,16 +215,19 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit probeCopy -> m.state )))(this) def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) - def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt) = { - val popped = m.sharers.pop(src) - val next = MasterMetadata(Mux(popped.none(), masterInvalid, masterValid), popped)(this) - def is(r: UInt) = incoming.r_type === r + def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { + val next = MasterMetadata(masterValid, m.sharers.pop(src))(this) MuxBundle(m, Array( - is(releaseVoluntaryInvalidateData) -> next, - is(releaseInvalidateData) -> next, - is(releaseCopyData) -> m, - is(releaseInvalidateAck) -> next, - is(releaseCopyAck) -> m + r.is(releaseVoluntaryInvalidateData) -> next, + r.is(releaseInvalidateData) -> next, + r.is(releaseInvalidateAck) -> next + )) + } + def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { + val cached = MasterMetadata(masterValid, m.sharers.push(dst))(this) + val uncached = MasterMetadata(masterValid, m.sharers)(this) + MuxBundle(uncached, Array( + g.is(grantReadExclusive) -> cached )) } @@ -292,6 +296,7 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit acquireAtomicUncached -> probeInvalidate )) } + def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate def requiresOuterRead(a_type: UInt) = { (a_type != acquireWriteUncached) @@ -299,11 +304,10 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit def requiresOuterWrite(a_type: UInt) = { (a_type === acquireWriteUncached) } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached + def requiresProbes(a_type: UInt, m: MasterMetadata) = !m.sharers.none() def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -346,6 +350,7 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def needsWriteback (m: ClientMetadata): Bool = { needsTransactionOnCacheControl(M_INV, m) } + def needsWriteback(m: MasterMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state))(this) @@ -374,18 +379,20 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi probeCopy -> m.state )))(this) def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) - def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt) = { - val popped = m.sharers.pop(src) - val next = MasterMetadata(Mux(popped.none(), masterInvalid, masterValid), popped)(this) - def is(r: UInt) = incoming.r_type === r + def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { + val next = MasterMetadata(masterValid, m.sharers.pop(src))(this) MuxBundle(m, Array( - is(releaseVoluntaryInvalidateData) -> next, - is(releaseInvalidateData) -> next, - is(releaseDowngradeData) -> m, - is(releaseCopyData) -> m, - is(releaseInvalidateAck) -> next, - is(releaseDowngradeAck) -> m, - is(releaseCopyAck) -> m + r.is(releaseVoluntaryInvalidateData) -> next, + r.is(releaseInvalidateData) -> next, + r.is(releaseInvalidateAck) -> next + )) + } + def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { + val cached = MasterMetadata(masterValid, m.sharers.push(dst))(this) + val uncached = MasterMetadata(masterValid, m.sharers)(this) + MuxBundle(uncached, Array( + g.is(grantReadExclusive) -> cached, + g.is(grantReadExclusiveAck) -> cached )) } @@ -462,6 +469,7 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi acquireAtomicUncached -> probeInvalidate )) } + def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate def requiresOuterRead(a_type: UInt) = { (a_type != acquireWriteUncached) @@ -469,25 +477,25 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def requiresOuterWrite(a_type: UInt) = { (a_type === acquireWriteUncached) } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached + def requiresProbes(a_type: UInt, m: MasterMetadata) = !m.sharers.none() def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 3 - def nMasterStates = 3 + def nMasterStates = 2 def nAcquireTypes = 7 def nProbeTypes = 3 def nReleaseTypes = 7 def nGrantTypes = 9 val clientInvalid :: clientShared :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) + //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) + val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) @@ -522,6 +530,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def needsWriteback (m: ClientMetadata): Bool = { needsTransactionOnCacheControl(M_INV, m) } + def needsWriteback(m: MasterMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state))(this) @@ -549,20 +558,21 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi probeCopy -> m.state )))(this) def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) - def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt) = { - val popped = m.sharers.pop(src) - val next = MasterMetadata( - Mux(popped.none(), masterInvalid, - Mux(popped.one(), masterExclusive, masterShared)), popped)(this) - def is(r: UInt) = incoming.r_type === r + def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { + val next = MasterMetadata(masterValid, m.sharers.pop(src))(this) MuxBundle(m, Array( - is(releaseVoluntaryInvalidateData) -> next, - is(releaseInvalidateData) -> next, - is(releaseDowngradeData) -> m, - is(releaseCopyData) -> m, - is(releaseInvalidateAck) -> next, - is(releaseDowngradeAck) -> m, - is(releaseCopyAck) -> m + r.is(releaseVoluntaryInvalidateData) -> next, + r.is(releaseInvalidateData) -> next, + r.is(releaseInvalidateAck) -> next + )) + } + def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { + val cached = MasterMetadata(masterValid, m.sharers.push(dst))(this) + val uncached = MasterMetadata(masterValid, m.sharers)(this) + MuxBundle(uncached, Array( + g.is(grantReadShared) -> cached, + g.is(grantReadExclusive) -> cached, + g.is(grantReadExclusiveAck) -> cached )) } @@ -612,7 +622,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def getGrantType(a: Acquire, m: MasterMetadata): UInt = { MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), + acquireReadShared -> Mux(!m.sharers.none(), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -635,6 +645,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi acquireWriteUncached -> probeInvalidate )) } + def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate def requiresOuterRead(a_type: UInt) = { (a_type != acquireWriteUncached) @@ -642,25 +653,25 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def requiresOuterWrite(a_type: UInt) = { (a_type === acquireWriteUncached) } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached + def requiresProbes(a_type: UInt, m: MasterMetadata) = !m.sharers.none() def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 4 - def nMasterStates = 3 + def nMasterStates = 2 def nAcquireTypes = 7 def nProbeTypes = 3 def nReleaseTypes = 7 def nGrantTypes = 9 val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) + //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) + val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) @@ -695,6 +706,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW def needsWriteback (m: ClientMetadata): Bool = { needsTransactionOnCacheControl(M_INV, m) } + def needsWriteback(m: MasterMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state))(this) @@ -723,20 +735,21 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW probeCopy -> m.state )))(this) def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) - def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt) = { - val popped = m.sharers.pop(src) - val next = MasterMetadata( - Mux(popped.none(), masterInvalid, - Mux(popped.one(), masterExclusive, masterShared)), popped)(this) - def is(r: UInt) = incoming.r_type === r + def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { + val next = MasterMetadata(masterValid, m.sharers.pop(src))(this) MuxBundle(m, Array( - is(releaseVoluntaryInvalidateData) -> next, - is(releaseInvalidateData) -> next, - is(releaseDowngradeData) -> m, - is(releaseCopyData) -> m, - is(releaseInvalidateAck) -> next, - is(releaseDowngradeAck) -> m, - is(releaseCopyAck) -> m + r.is(releaseVoluntaryInvalidateData) -> next, + r.is(releaseInvalidateData) -> next, + r.is(releaseInvalidateAck) -> next + )) + } + def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { + val cached = MasterMetadata(masterValid, m.sharers.push(dst))(this) + val uncached = MasterMetadata(masterValid, m.sharers)(this) + MuxBundle(uncached, Array( + g.is(grantReadShared) -> cached, + g.is(grantReadExclusive) -> cached, + g.is(grantReadExclusiveAck) -> cached )) } @@ -786,7 +799,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW def getGrantType(a: Acquire, m: MasterMetadata): UInt = { MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), + acquireReadShared -> Mux(!m.sharers.none(), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -813,6 +826,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW acquireAtomicUncached -> probeInvalidate )) } + def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate def requiresOuterRead(a_type: UInt) = { (a_type != acquireWriteUncached) @@ -820,26 +834,26 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW def requiresOuterWrite(a_type: UInt) = { (a_type === acquireWriteUncached) } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached + def requiresProbes(a_type: UInt, m: MasterMetadata) = !m.sharers.none() def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 7 - def nMasterStates = 3 + def nMasterStates = 2 def nAcquireTypes = 8 def nProbeTypes = 4 def nReleaseTypes = 11 def nGrantTypes = 9 val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: clientSharedByTwo :: clientMigratoryClean :: clientMigratoryDirty :: Nil = Enum(UInt(), nClientStates) - val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) + //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) + val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(UInt(), nProbeTypes) @@ -873,6 +887,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def needsWriteback (m: ClientMetadata): Bool = { needsTransactionOnCacheControl(M_INV, m) } + def needsWriteback(m: MasterMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata( Mux(isWrite(cmd), MuxLookup(m.state, clientExclusiveDirty, Array( @@ -914,25 +929,24 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo clientMigratoryDirty -> clientInvalid)) )))(this) def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) - def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt) = { - val popped = m.sharers.pop(src) - val next = MasterMetadata( - Mux(popped.none(), masterInvalid, - Mux(popped.one(), masterExclusive, masterShared)), - popped)(this) - def is(r: UInt) = incoming.r_type === r + def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { + val next = MasterMetadata(masterValid, m.sharers.pop(src))(this) MuxBundle(m, Array( - is(releaseVoluntaryInvalidateData) -> next, - is(releaseInvalidateData) -> next, - is(releaseDowngradeData) -> m, - is(releaseCopyData) -> m, - is(releaseInvalidateAck) -> next, - is(releaseDowngradeAck) -> m, - is(releaseCopyAck) -> m, - is(releaseDowngradeDataMigratory) -> m, - is(releaseDowngradeAckHasCopy) -> m, - is(releaseInvalidateDataMigratory) -> next, - is(releaseInvalidateAckMigratory) -> next + r.is(releaseVoluntaryInvalidateData) -> next, + r.is(releaseInvalidateData) -> next, + r.is(releaseInvalidateAck) -> next, + r.is(releaseInvalidateDataMigratory) -> next, + r.is(releaseInvalidateAckMigratory) -> next + )) + } + def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { + val cached = MasterMetadata(masterValid, m.sharers.push(dst))(this) + val uncached = MasterMetadata(masterValid, m.sharers)(this) + MuxBundle(uncached, Array( + g.is(grantReadShared) -> cached, + g.is(grantReadExclusive) -> cached, + g.is(grantReadExclusiveAck) -> cached, + g.is(grantReadMigratory) -> cached )) } @@ -983,7 +997,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def getGrantType(a: Acquire, m: MasterMetadata): UInt = { MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? + acquireReadShared -> Mux(!m.sharers.none(), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -1012,6 +1026,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo acquireInvalidateOthers -> probeInvalidateOthers )) } + def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate def requiresOuterRead(a_type: UInt) = { (a_type != acquireWriteUncached && a_type != acquireInvalidateOthers) @@ -1019,12 +1034,11 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def requiresOuterWrite(a_type: UInt) = { (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached || a_type === acquireAtomicUncached) } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached + def requiresProbes(a_type: UInt, m: MasterMetadata) = !m.sharers.none() def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 51864fe1..46ae899e 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -77,6 +77,7 @@ class Acquire extends ClientSourcedMessage val write_mask = Bits(width = params(TLWriteMaskBits)) val subword_addr = Bits(width = params(TLWordAddrBits)) val atomic_opcode = Bits(width = params(TLAtomicOpBits)) + def is(t: UInt) = a_type === t } object Probe @@ -94,6 +95,7 @@ class Probe extends MasterSourcedMessage with HasPhysicalAddress with HasMasterTransactionId { val p_type = UInt(width = params(TLCoherence).probeTypeWidth) + def is(t: UInt) = p_type === t } object Release @@ -127,6 +129,7 @@ class Release extends ClientSourcedMessage with HasMasterTransactionId with HasTileLinkData { val r_type = UInt(width = params(TLCoherence).releaseTypeWidth) + def is(t: UInt) = r_type === t } object Grant @@ -151,6 +154,7 @@ class Grant extends MasterSourcedMessage with HasClientTransactionId with HasMasterTransactionId { val g_type = UInt(width = params(TLCoherence).grantTypeWidth) + def is(t: UInt) = g_type === t } class Finish extends ClientSourcedMessage with HasMasterTransactionId From 044b19dbc1de7dfeb1d9c15a76e937ef4bdc0a0c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 15 Oct 2014 11:46:35 -0700 Subject: [PATCH 255/374] Compiles and elaborates, does not pass asm tests --- uncore/src/main/scala/cache.scala | 31 +++++---- uncore/src/main/scala/coherence.scala | 93 +++++++++++++-------------- 2 files changed, 63 insertions(+), 61 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 9cc99e4e..1c6fc044 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -130,8 +130,9 @@ class L2MetaReadReq extends MetaReadReq with HasL2Id { } class L2MetaWriteReq extends MetaWriteReq[L2Metadata](new L2Metadata) - with HasL2Id - + with HasL2Id { + override def clone = new L2MetaWriteReq().asInstanceOf[this.type] +} class L2MetaResp extends L2HellaCacheBundle with HasL2Id with HasL2InternalRequestState @@ -360,7 +361,7 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa } class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_meta_read :: s_meta_resp :: s_meta_write :: s_data_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 6) + val s_idle :: s_meta_read :: s_meta_resp :: s_meta_write :: s_data_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 7) val state = Reg(init=s_idle) val xact = Reg{ new Release } val xact_internal = Reg{ new L2MetaResp } @@ -399,7 +400,6 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) io.meta_write.bits.way_en := xact_internal.way_en io.meta_write.bits.data := xact_internal.meta - io.meta_resp.valid := Bool(true) switch (state) { is(s_idle) { @@ -450,21 +450,22 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val state = Reg(init=s_idle) val xact = Reg{ new Acquire } val xact_internal = Reg{ new L2MetaResp } + val test = Reg{UInt()} val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) //TODO: Will need id reg for merged release xacts val release_count = Reg(init = UInt(0, width = log2Up(nClients))) - val pending_probes = Reg(init = co.dir()) - val curr_p_id = pending_probes.next() + val pending_probes = Reg(init = co.dir().flush) + val curr_p_id = co.dir().next(pending_probes) val is_uncached = co.messageIsUncached(xact) val tag_match = xact_internal.tag_match val needs_writeback = co.needsWriteback(xact_internal.meta.coh) val is_hit = co.isHit(xact, xact_internal.meta.coh) val needs_probes = co.requiresProbes(xact.a_type, xact_internal.meta.coh) - val c_rel_had_data = Reg{Bool()} - val c_rel_was_voluntary = Reg{Bool()} + val c_rel_had_data = Reg(init = Bool(false)) + val c_rel_was_voluntary = Reg(init = Bool(false)) val wb_buffer = Reg{xact.data.clone} io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && @@ -557,12 +558,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val _is_hit = co.isHit(xact, coh) val _needs_probes = co.requiresProbes(xact.a_type, coh) xact_internal := io.meta_resp.bits + test := UInt(0) when(!_needs_writeback) { - xact_internal.meta.coh := co.masterMetadataOnFlush +// xact_internal.meta.coh := co.masterMetadataOnFlush + test := UInt(12) } when(_needs_probes) { pending_probes := coh.sharers - release_count := coh.sharers.count() + release_count := co.dir().count(coh.sharers) c_rel_had_data := Bool(false) c_rel_was_voluntary := Bool(false) } @@ -579,19 +582,21 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val skip = io.tile_incoherent(curr_p_id) || ((curr_p_id === init_client_id) && !co.requiresSelfProbe(xact.a_type)) - io.inner.probe.valid := !(pending_probes.none() || skip) + io.inner.probe.valid := !(co.dir().none(pending_probes) || skip) when(io.inner.probe.ready || skip) { - pending_probes.pop(curr_p_id) + co.dir().pop(pending_probes, curr_p_id) } when(skip) { release_count := release_count - UInt(1) } // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) when(io.inner.release.valid) { +/* xact_internal.meta.coh := co.masterMetadataOnRelease( c_rel.payload, xact_internal.meta.coh, c_rel.header.src) +*/ when(co.messageHasData(c_rel.payload)) { c_rel_had_data := Bool(true) when(tag_match) { @@ -623,7 +628,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } is(s_data_resp_wb) { when(io.data_resp.valid) { - wb_buffer := io.data_resp.bits + wb_buffer := io.data_resp.bits.data state := s_outer_write_wb } } diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 5dd87b35..2bb6ec01 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -28,10 +28,10 @@ object MasterMetadata { def apply(state: UInt)(implicit c: CoherencePolicy): MasterMetadata = { val m = new MasterMetadata m.state := state - m.sharers.flush() + m.sharers := c.dir().flush m } - def apply(state: UInt, sharers: DirectoryRepresentation)(implicit c: CoherencePolicy): MasterMetadata = { + def apply(state: UInt, sharers: UInt)(implicit c: CoherencePolicy): MasterMetadata = { val m = apply(state) m.sharers := sharers m @@ -39,7 +39,7 @@ object MasterMetadata { } class MasterMetadata(implicit c: CoherencePolicy) extends CoherenceMetadata { val state = UInt(width = c.masterStateWidth) - val sharers = c.dir() + val sharers = UInt(width = c.dir().width) override def clone = new MasterMetadata()(c).asInstanceOf[this.type] } /* @@ -50,37 +50,34 @@ class MixedMetadata(inner: CoherencePolicy, outer: CoherencePolicy) extends Cohe } */ -abstract class DirectoryRepresentation extends Bundle { - def pop(id: UInt): DirectoryRepresentation - def push(id: UInt): DirectoryRepresentation - def flush(dummy: Int = 0): DirectoryRepresentation - def none(dummy: Int = 0): Bool - def one(dummy: Int = 0): Bool - def count(dummy: Int = 0): UInt - def next(dummy: Int = 0): UInt +abstract class DirectoryRepresentation(val width: Int) { + def pop(prev: UInt, id: UInt): UInt + def push(prev: UInt, id: UInt): UInt + def flush: UInt + def none(s: UInt): Bool + def one(s: UInt): Bool + def count(s: UInt): UInt + def next(s: UInt): UInt } -class NullRepresentation extends DirectoryRepresentation { - val internal = UInt() - def pop(id: UInt) = this - def push(id: UInt) = this - def flush(dummy: Int = 0) = { internal := UInt(0); this } - def none(dummy: Int = 0) = Bool(false) - def one(dummy: Int = 0) = Bool(false) - def count(dummy: Int = 0) = UInt(0) - def next(dummy: Int = 0) = UInt(0) +class NullRepresentation extends DirectoryRepresentation(1) { + def pop(prev: UInt, id: UInt) = UInt(0) + def push(prev: UInt, id: UInt) = UInt(0) + def flush = UInt(0) + def none(s: UInt) = Bool(false) + def one(s: UInt) = Bool(false) + def count(s: UInt) = UInt(0) + def next(s: UInt) = UInt(0) } -class FullRepresentation(nClients: Int) extends DirectoryRepresentation { - val internal = UInt(width = nClients) - def pop(id: UInt) = { internal := internal & ~UIntToOH(id); this } // make new FullRep to return? - def push(id: UInt) = { internal := internal | UIntToOH(id); this } - def flush(dummy: Int = 0) = { internal := UInt(0, width = nClients); this } - def none(dummy: Int = 0) = internal === UInt(0) - def one(dummy: Int = 0) = PopCount(internal) === UInt(1) - def count(dummy: Int = 0) = PopCount(internal) - def next(dummy: Int = 0) = PriorityEncoder(internal) - override def clone = new FullRepresentation(nClients).asInstanceOf[this.type] +class FullRepresentation(nClients: Int) extends DirectoryRepresentation(nClients) { + def pop(prev: UInt, id: UInt) = prev & ~UIntToOH(id) + def push(prev: UInt, id: UInt) = prev | UIntToOH(id) + def flush = UInt(0, width = width) + def none(s: UInt) = s === UInt(0) + def one(s: UInt) = PopCount(s) === UInt(1) + def count(s: UInt) = PopCount(s) + def next(s: UInt) = PriorityEncoder(s) } abstract class CoherencePolicy(val dir: () => DirectoryRepresentation) { @@ -216,7 +213,7 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit )))(this) def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, m.sharers.pop(src))(this) + val next = MasterMetadata(masterValid, dir().pop(m.sharers, src))(this) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -224,7 +221,7 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit )) } def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, m.sharers.push(dst))(this) + val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) val uncached = MasterMetadata(masterValid, m.sharers)(this) MuxBundle(uncached, Array( g.is(grantReadExclusive) -> cached @@ -307,7 +304,7 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached - def requiresProbes(a_type: UInt, m: MasterMetadata) = !m.sharers.none() + def requiresProbes(a_type: UInt, m: MasterMetadata) = !dir().none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -380,7 +377,7 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi )))(this) def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, m.sharers.pop(src))(this) + val next = MasterMetadata(masterValid, dir().pop(m.sharers,src))(this) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -388,7 +385,7 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi )) } def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, m.sharers.push(dst))(this) + val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) val uncached = MasterMetadata(masterValid, m.sharers)(this) MuxBundle(uncached, Array( g.is(grantReadExclusive) -> cached, @@ -480,7 +477,7 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached - def requiresProbes(a_type: UInt, m: MasterMetadata) = !m.sharers.none() + def requiresProbes(a_type: UInt, m: MasterMetadata) = !dir().none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -559,7 +556,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi )))(this) def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, m.sharers.pop(src))(this) + val next = MasterMetadata(masterValid, dir().pop(m.sharers,src))(this) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -567,7 +564,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi )) } def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, m.sharers.push(dst))(this) + val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) val uncached = MasterMetadata(masterValid, m.sharers)(this) MuxBundle(uncached, Array( g.is(grantReadShared) -> cached, @@ -622,7 +619,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def getGrantType(a: Acquire, m: MasterMetadata): UInt = { MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadShared -> Mux(!m.sharers.none(), grantReadShared, grantReadExclusive), + acquireReadShared -> Mux(!dir().none(m.sharers), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -656,7 +653,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached - def requiresProbes(a_type: UInt, m: MasterMetadata) = !m.sharers.none() + def requiresProbes(a_type: UInt, m: MasterMetadata) = !dir().none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -736,7 +733,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW )))(this) def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, m.sharers.pop(src))(this) + val next = MasterMetadata(masterValid, dir().pop(m.sharers,src))(this) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -744,7 +741,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW )) } def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, m.sharers.push(dst))(this) + val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) val uncached = MasterMetadata(masterValid, m.sharers)(this) MuxBundle(uncached, Array( g.is(grantReadShared) -> cached, @@ -799,7 +796,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW def getGrantType(a: Acquire, m: MasterMetadata): UInt = { MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadShared -> Mux(!m.sharers.none(), grantReadShared, grantReadExclusive), + acquireReadShared -> Mux(!dir().none(m.sharers), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -838,7 +835,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached - def requiresProbes(a_type: UInt, m: MasterMetadata) = !m.sharers.none() + def requiresProbes(a_type: UInt, m: MasterMetadata) = !dir().none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -930,7 +927,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo )))(this) def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, m.sharers.pop(src))(this) + val next = MasterMetadata(masterValid, dir().pop(m.sharers,src))(this) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -940,7 +937,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo )) } def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, m.sharers.push(dst))(this) + val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) val uncached = MasterMetadata(masterValid, m.sharers)(this) MuxBundle(uncached, Array( g.is(grantReadShared) -> cached, @@ -997,7 +994,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def getGrantType(a: Acquire, m: MasterMetadata): UInt = { MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadShared -> Mux(!m.sharers.none(), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? + acquireReadShared -> Mux(!dir().none(m.sharers), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -1038,7 +1035,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached - def requiresProbes(a_type: UInt, m: MasterMetadata) = !m.sharers.none() + def requiresProbes(a_type: UInt, m: MasterMetadata) = !dir().none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } From a891ba1d460c832d97d8e5b6e7865344eb997755 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 21 Oct 2014 17:40:30 -0700 Subject: [PATCH 256/374] more correct handling of internal state --- uncore/src/main/scala/cache.scala | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 1c6fc044..4ca6b53e 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -459,11 +459,22 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val pending_probes = Reg(init = co.dir().flush) val curr_p_id = co.dir().next(pending_probes) + + val next_coh_on_release = co.masterMetadataOnRelease( + c_rel.payload, + xact_internal.meta.coh, + c_rel.header.src) + val next_coh_on_grant = co.masterMetadataOnGrant( + c_gnt.payload, + xact_internal.meta.coh, + c_gnt.header.dst) val is_uncached = co.messageIsUncached(xact) val tag_match = xact_internal.tag_match val needs_writeback = co.needsWriteback(xact_internal.meta.coh) val is_hit = co.isHit(xact, xact_internal.meta.coh) val needs_probes = co.requiresProbes(xact.a_type, xact_internal.meta.coh) + //TODO: does allocate + val c_rel_had_data = Reg(init = Bool(false)) val c_rel_was_voluntary = Reg(init = Bool(false)) val wb_buffer = Reg{xact.data.clone} @@ -535,7 +546,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) io.meta_write.bits.way_en := xact_internal.way_en io.meta_write.bits.data.tag := xact.addr >> UInt(untagBits) - io.meta_write.bits.data.coh := xact_internal.meta.coh + io.meta_write.bits.data.coh := next_coh_on_grant switch (state) { is(s_idle) { @@ -558,11 +569,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val _is_hit = co.isHit(xact, coh) val _needs_probes = co.requiresProbes(xact.a_type, coh) xact_internal := io.meta_resp.bits - test := UInt(0) - when(!_needs_writeback) { -// xact_internal.meta.coh := co.masterMetadataOnFlush - test := UInt(12) - } when(_needs_probes) { pending_probes := coh.sharers release_count := co.dir().count(coh.sharers) @@ -591,12 +597,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) when(io.inner.release.valid) { -/* - xact_internal.meta.coh := co.masterMetadataOnRelease( - c_rel.payload, - xact_internal.meta.coh, - c_rel.header.src) -*/ + xact_internal.meta.coh := next_coh_on_release when(co.messageHasData(c_rel.payload)) { c_rel_had_data := Bool(true) when(tag_match) { From 27c72e5eedd708fe98c73aabd39124f48a099177 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 23 Oct 2014 21:50:03 -0700 Subject: [PATCH 257/374] nearly all isa tests pass --- uncore/src/main/scala/cache.scala | 152 ++++++++++++++------------ uncore/src/main/scala/coherence.scala | 14 +++ uncore/src/main/scala/memserdes.scala | 2 +- 3 files changed, 97 insertions(+), 71 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 4ca6b53e..6b5e9ba8 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -157,13 +157,13 @@ class L2MetadataArray extends L2HellaCacheModule { val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) val s2_tag_match = s2_tag_match_way.orR val s2_hit_coh = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).coh, s1_clk_en))) - //val s2_hit = s2_tag_match && tl.co.isHit(s2_req.cmd, s2_hit_state) && s2_hit_state === tl.co.newStateOnHit(s2_req.cmd, s2_hit_state) val replacer = params(Replacer)() val s1_replaced_way_en = UIntToOH(replacer.way) val s2_replaced_way_en = UIntToOH(RegEnable(replacer.way, s1_clk_en)) val s2_repl_meta = Mux1H(s2_replaced_way_en, wayMap((w: Int) => RegEnable(meta.io.resp(w), s1_clk_en && s1_replaced_way_en(w))).toSeq) + when(!s2_tag_match) { replacer.miss } io.resp.valid := Reg(next = s1_clk_en) io.resp.bits.id := RegEnable(s1_id, s1_clk_en) @@ -197,9 +197,9 @@ class L2DataArray extends L2HellaCacheModule { val waddr = io.write.bits.addr val raddr = io.read.bits.addr - val wmask = FillInterleaved(wordBits, io.write.bits.wmask) + val wmask = FillInterleaved(8, io.write.bits.wmask) val resp = (0 until nWays).map { w => - val array = Mem(Bits(width=params(RowBits)), nSets*refillCycles, seqRead = true) + val array = Mem(Bits(width=rowBits), nSets*refillCycles, seqRead = true) when (io.write.bits.way_en(w) && io.write.valid) { array.write(waddr, io.write.bits.data, wmask) } @@ -258,7 +258,7 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac def doInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { outs.map(_.bits := in.bits) - outs.zipWithIndex.map { case (o, i) => o.valid := UInt(i) === in.bits.id } + outs.zipWithIndex.map { case (o, i) => o.valid := in.valid && (UInt(i) === in.bits.id) } } // Create TSHRs for outstanding transactions @@ -295,8 +295,7 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) val block_releases = Bool(false) val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b} - //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response - val release_idx = Mux(voluntary, UInt(0), release.bits.payload.master_xact_id) + val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.inner t.release.bits := release.bits @@ -361,7 +360,7 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa } class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_meta_read :: s_meta_resp :: s_meta_write :: s_data_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 7) + val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 7) val state = Reg(init=s_idle) val xact = Reg{ new Release } val xact_internal = Reg{ new L2MetaResp } @@ -373,11 +372,12 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.outer.grant.ready := Bool(false) io.outer.acquire.valid := Bool(false) - io.inner.acquire.ready := Bool(false) io.inner.probe.valid := Bool(false) io.inner.release.ready := Bool(false) io.inner.grant.valid := Bool(false) + io.inner.finish.ready := Bool(false) + io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id io.inner.grant.bits.payload := Grant(co.getGrantType(xact, xact_internal.meta.coh), @@ -399,7 +399,10 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.meta_write.bits.id := UInt(trackerId) io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) io.meta_write.bits.way_en := xact_internal.way_en - io.meta_write.bits.data := xact_internal.meta + io.meta_write.bits.data.tag := xact.addr >> UInt(untagBits) + io.meta_write.bits.data.coh := co.masterMetadataOnRelease(xact, + xact_internal.meta.coh, + init_client_id) switch (state) { is(s_idle) { @@ -416,9 +419,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou } is(s_meta_resp) { when(io.meta_resp.valid) { - xact_internal := co.masterMetadataOnRelease(xact, - io.meta_resp.bits.meta.coh, - init_client_id) + xact_internal := io.meta_resp.bits state := Mux(io.meta_resp.bits.tag_match, Mux(co.messageHasData(xact), s_data_write, s_meta_write), s_grant) @@ -440,6 +441,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou } } is(s_busy) { + io.inner.finish.ready := Bool(true) when(io.inner.finish.valid) { state := s_idle } } } @@ -450,16 +452,29 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val state = Reg(init=s_idle) val xact = Reg{ new Acquire } val xact_internal = Reg{ new L2MetaResp } - val test = Reg{UInt()} - val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) - //TODO: Will need id reg for merged release xacts - + val init_client_id = Reg(init = UInt(0, width = log2Up(nClients))) + val crel_had_data = Reg(init = Bool(false)) + val crel_was_voluntary = Reg(init = Bool(false)) + val crel_wb_src = Reg(init = UInt(0, width = log2Up(nClients))) + val crel_wb_g_type = Reg(init = UInt(0, width = co.grantTypeWidth)) + val wb_buffer = Reg{xact.data.clone} - val release_count = Reg(init = UInt(0, width = log2Up(nClients))) + val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) val pending_probes = Reg(init = co.dir().flush) val curr_p_id = co.dir().next(pending_probes) - + val is_uncached = co.messageIsUncached(xact) + val tag_match = xact_internal.tag_match + val needs_writeback = !tag_match && co.needsWriteback(xact_internal.meta.coh) + val is_hit = tag_match && co.isHit(xact, xact_internal.meta.coh) + val needs_probes = co.requiresProbes(xact.a_type, xact_internal.meta.coh) + //TODO: does allocate + + io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && + (state != s_idle) //TODO: Also indexes + io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && + (state != s_idle) //TODO: Also indexes? + val next_coh_on_release = co.masterMetadataOnRelease( c_rel.payload, xact_internal.meta.coh, @@ -468,29 +483,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St c_gnt.payload, xact_internal.meta.coh, c_gnt.header.dst) - val is_uncached = co.messageIsUncached(xact) - val tag_match = xact_internal.tag_match - val needs_writeback = co.needsWriteback(xact_internal.meta.coh) - val is_hit = co.isHit(xact, xact_internal.meta.coh) - val needs_probes = co.requiresProbes(xact.a_type, xact_internal.meta.coh) - //TODO: does allocate - - val c_rel_had_data = Reg(init = Bool(false)) - val c_rel_was_voluntary = Reg(init = Bool(false)) - val wb_buffer = Reg{xact.data.clone} - - io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && - (state != s_idle) //TODO: Also indexes - io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && - (state != s_idle) //TODO: Also indexes? + val addr_wb = Cat(xact_internal.meta.tag, xact.addr(untagBits-1,blockOffBits)) val outer_write_acq = Bundle(Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId), xact.data), { case TLId => outerId }) val outer_write_wb = Bundle(Acquire(co.getUncachedWriteAcquireType, - Cat(xact_internal.meta.tag, - xact.addr(untagBits-1,blockOffBits)), + addr_wb, UInt(trackerId), wb_buffer), { case TLId => outerId }) val outer_read = Bundle(Acquire(co.getUncachedReadAcquireType, @@ -501,31 +501,33 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.outer.acquire.bits.header.src := UInt(bankId) io.outer.grant.ready := Bool(true) //grant.data -> xact.data - val inner_probe_cacq = Probe(co.getProbeType(xact, xact_internal.meta.coh), + val cprb_for_cacq = Probe(co.getProbeType(xact, xact_internal.meta.coh), xact.addr, UInt(trackerId)) - val inner_probe_wb = Probe(co.getProbeTypeOnVoluntaryWriteback, - xact.addr, + val cprb_for_mwb = Probe(co.getProbeTypeOnVoluntaryWriteback, + addr_wb, UInt(trackerId)) //TODO inner_probe_mprb io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Mux(needs_writeback, - inner_probe_wb, - inner_probe_cacq) + io.inner.probe.bits.payload := Mux(!tag_match && needs_writeback, + cprb_for_mwb, + cprb_for_cacq) - val grant_type = co.getGrantType(xact, xact_internal.meta.coh) + val cgnt_for_cacq = Grant(co.getGrantType(xact, xact_internal.meta.coh), + xact.client_xact_id, + UInt(trackerId), + xact.data) + val cgnt_for_cwb = Grant(crel_wb_g_type, UInt(0), UInt(trackerId), UInt(0)) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := init_client_id - io.inner.grant.bits.payload := Grant(grant_type, - xact.client_xact_id, - UInt(trackerId), - xact.data) + io.inner.grant.bits.header.dst := Mux(crel_was_voluntary, crel_wb_src, init_client_id) + io.inner.grant.bits.payload := Mux(crel_was_voluntary, cgnt_for_cwb, cgnt_for_cacq) io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) + io.inner.finish.ready := Bool(false) io.data_read.valid := Bool(false) io.data_read.bits.id := UInt(trackerId) @@ -565,15 +567,18 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St when(io.meta_resp.valid) { val coh = io.meta_resp.bits.meta.coh val _tag_match = io.meta_resp.bits.tag_match - val _needs_writeback = co.needsWriteback(coh) - val _is_hit = co.isHit(xact, coh) + val _needs_writeback = !_tag_match && co.needsWriteback(coh) + val _is_hit = _tag_match && co.isHit(xact, coh) val _needs_probes = co.requiresProbes(xact.a_type, coh) xact_internal := io.meta_resp.bits when(_needs_probes) { - pending_probes := coh.sharers - release_count := co.dir().count(coh.sharers) - c_rel_had_data := Bool(false) - c_rel_was_voluntary := Bool(false) + val mask_incoherent = co.dir().full(coh.sharers) & ~io.tile_incoherent + val mask_self = mask_incoherent & + ~(!(co.requiresSelfProbe(xact.a_type) || _needs_writeback) << init_client_id) + pending_probes := mask_self + release_count := co.dir().count(mask_self) + crel_had_data := Bool(false) + crel_was_voluntary := Bool(false) } state := Mux(_tag_match, Mux(_is_hit, @@ -585,21 +590,17 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } } is(s_probe) { - val skip = io.tile_incoherent(curr_p_id) || - ((curr_p_id === init_client_id) && - !co.requiresSelfProbe(xact.a_type)) - io.inner.probe.valid := !(co.dir().none(pending_probes) || skip) - when(io.inner.probe.ready || skip) { - co.dir().pop(pending_probes, curr_p_id) + io.inner.probe.valid := !co.dir().none(pending_probes) + when(io.inner.probe.ready) { + pending_probes := co.dir().pop(pending_probes, curr_p_id) } - when(skip) { release_count := release_count - UInt(1) } // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) when(io.inner.release.valid) { xact_internal.meta.coh := next_coh_on_release when(co.messageHasData(c_rel.payload)) { - c_rel_had_data := Bool(true) + crel_had_data := Bool(true) when(tag_match) { xact.data := mergeData(xact, io.inner.release.bits.payload.data) } .otherwise { @@ -607,19 +608,29 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } } when(co.isVoluntary(c_rel.payload)) { - c_rel_was_voluntary := Bool(true) + crel_was_voluntary := Bool(true) + crel_wb_src := c_rel.header.src + crel_wb_g_type := co.getGrantType(c_rel.payload, xact_internal.meta.coh) } when(!co.isVoluntary(c_rel.payload)) { - release_count := release_count - Mux(skip, UInt(2), UInt(1)) - when(release_count === UInt(1)) { - state := Mux(tag_match, - Mux(is_hit, - Mux(c_rel_had_data, s_meta_write, s_data_read_hit), - s_outer_read), - Mux(c_rel_had_data, s_outer_write_wb, s_data_read_wb)) - } + release_count := release_count - UInt(1) } } + + when(crel_was_voluntary) { + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { + crel_was_voluntary := Bool(false) + } + } + + when(release_count === UInt(0) && !crel_was_voluntary) { + state := Mux(tag_match, + Mux(is_hit, + Mux(crel_had_data, s_data_write, s_data_read_hit), + s_outer_read), + Mux(crel_had_data, s_outer_write_wb, s_data_read_wb)) + } } is(s_data_read_wb) { io.data_read.valid := Bool(true) @@ -686,6 +697,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } } is(s_busy) { + io.inner.finish.ready := Bool(true) when(io.inner.finish.valid) { state := s_idle } } } diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 2bb6ec01..5afcccfb 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -58,6 +58,7 @@ abstract class DirectoryRepresentation(val width: Int) { def one(s: UInt): Bool def count(s: UInt): UInt def next(s: UInt): UInt + def full(s: UInt): UInt } class NullRepresentation extends DirectoryRepresentation(1) { @@ -68,6 +69,7 @@ class NullRepresentation extends DirectoryRepresentation(1) { def one(s: UInt) = Bool(false) def count(s: UInt) = UInt(0) def next(s: UInt) = UInt(0) + def full(s: UInt) = UInt(0) } class FullRepresentation(nClients: Int) extends DirectoryRepresentation(nClients) { @@ -78,6 +80,7 @@ class FullRepresentation(nClients: Int) extends DirectoryRepresentation(nClients def one(s: UInt) = PopCount(s) === UInt(1) def count(s: UInt) = PopCount(s) def next(s: UInt) = PriorityEncoder(s) + def full(s: UInt) = s } abstract class CoherencePolicy(val dir: () => DirectoryRepresentation) { @@ -147,6 +150,7 @@ trait UncachedTransactions { def getUncachedWriteWordAcquireType: UInt def getUncachedAtomicAcquireType: UInt def isUncachedReadTransaction(acq: Acquire): Bool + def getUncachedReadGrantType: UInt } abstract class CoherencePolicyWithUncached(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) @@ -234,6 +238,8 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit def getUncachedWriteWordAcquireType = acquireWriteWordUncached def getUncachedAtomicAcquireType = acquireAtomicUncached def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached + def getUncachedReadGrantType = grantReadUncached + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck @@ -399,6 +405,8 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def getUncachedWriteWordAcquireType = acquireWriteWordUncached def getUncachedAtomicAcquireType = acquireAtomicUncached def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached + def getUncachedReadGrantType = grantReadUncached + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck @@ -579,6 +587,8 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def getUncachedWriteWordAcquireType = acquireWriteWordUncached def getUncachedAtomicAcquireType = acquireAtomicUncached def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached + def getUncachedReadGrantType = grantReadUncached + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck @@ -756,6 +766,8 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW def getUncachedWriteWordAcquireType = acquireWriteWordUncached def getUncachedAtomicAcquireType = acquireAtomicUncached def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached + def getUncachedReadGrantType = grantReadUncached + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck @@ -954,6 +966,8 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def getUncachedWriteWordAcquireType = acquireWriteWordUncached def getUncachedAtomicAcquireType = acquireAtomicUncached def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached + def getUncachedReadGrantType = grantReadUncached + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index b470df49..fc4e0906 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -268,7 +268,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { // Aggregate incoming MemIO responses into TL Grants io.mem.resp.ready := !active_in || cnt_in < UInt(cnt_max) io.uncached.grant.valid := active_in && (cnt_in === UInt(cnt_max)) - io.uncached.grant.bits.payload := Grant(UInt(0), tag_in, UInt(0), buf_in) + io.uncached.grant.bits.payload := Grant(co.getUncachedReadGrantType, tag_in, UInt(0), buf_in) when(!active_in && io.mem.resp.valid) { active_in := Bool(true) cnt_in := UInt(1) From 10309849b7b7080b91281bf3cb5c95d5b5680900 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 5 Nov 2014 14:21:38 -0800 Subject: [PATCH 258/374] Remove master_xact_id from Probe and Release --- uncore/src/main/scala/tilelink.scala | 35 ++++++++++------------------ uncore/src/main/scala/uncore.scala | 8 +++---- 2 files changed, 15 insertions(+), 28 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 51864fe1..6738deaf 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -79,52 +79,41 @@ class Acquire extends ClientSourcedMessage val atomic_opcode = Bits(width = params(TLAtomicOpBits)) } -object Probe -{ - def apply(p_type: UInt, addr: UInt, master_xact_id: UInt) = { + +object Probe { + def apply(p_type: UInt, addr: UInt) = { val prb = new Probe prb.p_type := p_type prb.addr := addr - prb.master_xact_id := master_xact_id prb } } class Probe extends MasterSourcedMessage - with HasPhysicalAddress - with HasMasterTransactionId { + with HasPhysicalAddress { val p_type = UInt(width = params(TLCoherence).probeTypeWidth) } -object Release -{ - def apply(r_type: UInt, addr: UInt, data: UInt): Release = { - val rel = new Release - rel.r_type := r_type - rel.addr := addr - rel.data := data - rel - } - def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt): Release = { +object Release { + def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, data: UInt): Release = { val rel = new Release rel.r_type := r_type rel.addr := addr rel.client_xact_id := client_xact_id - rel.master_xact_id := master_xact_id - rel.data := UInt(0) - rel - } - def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt): Release = { - val rel = apply(r_type, addr, client_xact_id, master_xact_id) rel.data := data rel } + def apply(r_type: UInt, addr: UInt, client_xact_id: UInt): Release = { + apply(r_type, addr, client_xact_id, UInt(0)) + } + def apply(r_type: UInt, addr: UInt): Release = { + apply(r_type, addr, UInt(0), UInt(0)) + } } class Release extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId - with HasMasterTransactionId with HasTileLinkData { val r_type = UInt(width = params(TLCoherence).releaseTypeWidth) } diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 37e535f8..e1f59e08 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -62,8 +62,8 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) val block_releases = Bool(false) val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b} - //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response - val release_idx = Mux(voluntary, UInt(0), release.bits.payload.master_xact_id) + val release_idx = Mux(voluntary, UInt(0), conflict_idx) + // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.inner t.release.bits := release.bits @@ -200,9 +200,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.masterMetadataOnFlush), - xact.addr, - UInt(trackerId)) + io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.masterMetadataOnFlush), xact.addr) val grant_type = co.getGrantType(xact, co.masterMetadataOnFlush) io.inner.grant.valid := Bool(false) From 35553cc0b71ffe63b1ea0b9d775e691582376db5 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 11 Nov 2014 16:05:25 -0800 Subject: [PATCH 259/374] NullDirectory sharers.count fix --- uncore/src/main/scala/coherence.scala | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 7dbff4aa..9f94583a 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -51,7 +51,6 @@ class MixedMetadata(inner: CoherencePolicy, outer: CoherencePolicy) extends Cohe */ abstract class DirectoryRepresentation extends Bundle { - val internal: UInt def pop(id: UInt): DirectoryRepresentation def push(id: UInt): DirectoryRepresentation def flush(dummy: Int = 0): DirectoryRepresentation @@ -61,14 +60,13 @@ abstract class DirectoryRepresentation extends Bundle { def next(dummy: Int = 0): UInt } -class NullRepresentation extends DirectoryRepresentation { - val internal = UInt(0) +class NullRepresentation(nClients: Int) extends DirectoryRepresentation { def pop(id: UInt) = this def push(id: UInt) = this def flush(dummy: Int = 0) = this def none(dummy: Int = 0) = Bool(false) def one(dummy: Int = 0) = Bool(false) - def count(dummy: Int = 0) = UInt(0) + def count(dummy: Int = 0) = UInt(nClients) def next(dummy: Int = 0) = UInt(0) } From 82155f333e96bfb5eaa3b3b25ff7184609e39a17 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 11 Nov 2014 17:36:55 -0800 Subject: [PATCH 260/374] Major tilelink revision for uncached message types --- uncore/src/main/scala/coherence.scala | 566 ++++++++++---------------- uncore/src/main/scala/consts.scala | 2 + uncore/src/main/scala/htif.scala | 6 +- uncore/src/main/scala/memserdes.scala | 2 +- uncore/src/main/scala/tilelink.scala | 179 +++++--- uncore/src/main/scala/uncore.scala | 31 +- 6 files changed, 351 insertions(+), 435 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 9f94583a..a2af20cb 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -120,59 +120,48 @@ abstract class CoherencePolicy(val dir: () => DirectoryRepresentation) { def getReleaseTypeOnVoluntaryWriteback(): UInt def getReleaseTypeOnProbe(p: Probe, m: ClientMetadata): UInt def getGrantType(a: Acquire, m: MasterMetadata): UInt - def getGrantType(r: Release, m: MasterMetadata): UInt - //def getGrantType(a: Acquire) = getGrantType(a, new NullRepresentation) // TODO - //def getGrantType(r: Release) = getGrantType(r, new NullRepresentation) + def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt - def messageHasData (rel: SourcedMessage): Bool - def messageUpdatesDataArray (reply: Grant): Bool - def messageIsUncached(acq: Acquire): Bool + def messageHasData(rel: SourcedMessage): Bool + def messageUpdatesDataArray(g: Grant): Bool def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool def isVoluntary(rel: Release): Bool def isVoluntary(gnt: Grant): Bool - def requiresOuterRead(a_type: UInt, m: MasterMetadata): Bool - def requiresOuterWrite(a_type: UInt, m: MasterMetadata): Bool - def requiresOuterRead(a_type: UInt): Bool - def requiresOuterWrite(a_type: UInt): Bool - def requiresSelfProbe(a_type: UInt): Bool - def requiresAckForGrant(g_type: UInt): Bool - def requiresAckForRelease(r_type: UInt): Bool + def requiresOuterRead(acq: Acquire, m: MasterMetadata): Bool + def requiresOuterWrite(acq: Acquire, m: MasterMetadata): Bool + def requiresSelfProbe(a: Acquire): Bool + def requiresAckForGrant(g: Grant): Bool + def requiresAckForRelease(r: Release): Bool def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool + + def getGrantTypeForUncached(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, Grant.uncachedRead, Array( + Acquire.uncachedRead -> Grant.uncachedRead, + Acquire.uncachedWrite -> Grant.uncachedWrite, + Acquire.uncachedAtomic -> Grant.uncachedAtomic + )) + } } -trait UncachedTransactions { - def getUncachedReadAcquireType: UInt - def getUncachedWriteAcquireType: UInt - def getUncachedReadWordAcquireType: UInt - def getUncachedWriteWordAcquireType: UInt - def getUncachedAtomicAcquireType: UInt - def isUncachedReadTransaction(acq: Acquire): Bool -} - -abstract class CoherencePolicyWithUncached(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) - with UncachedTransactions - -class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 2 def nMasterStates = 2 - def nAcquireTypes = 6 + def nAcquireTypes = 1 def nProbeTypes = 2 def nReleaseTypes = 5 - def nGrantTypes = 7 + def nGrantTypes = 2 val clientInvalid :: clientValid :: Nil = Enum(UInt(), nClientStates) val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) - val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) + val acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) + val grantVoluntaryAck :: grantReadExclusive :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseCopyData) - val hasDataGrantTypeVec = Vec(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) + val hasDataGrantTypeVec = Vec(grantReadExclusive) def isHit (cmd: UInt, m: ClientMetadata): Bool = isValid(m) def isValid (m: ClientMetadata): Bool = m.state != clientInvalid @@ -197,15 +186,8 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit M_CLN -> clientValid )))(this) def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - MuxLookup(incoming.g_type, clientInvalid, Array( - grantReadExclusive -> clientValid, - grantReadUncached -> clientInvalid, - grantWriteUncached -> clientInvalid, - grantReadWordUncached -> clientInvalid, - grantWriteWordUncached -> clientInvalid, - grantAtomicUncached -> clientInvalid - )))(this) + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = + ClientMetadata(Mux(incoming.uncached, clientInvalid, clientValid))(this) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, @@ -225,14 +207,8 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit )) } - def getUncachedReadAcquireType = acquireReadUncached - def getUncachedWriteAcquireType = acquireWriteUncached - def getUncachedReadWordAcquireType = acquireReadWordUncached - def getUncachedWriteWordAcquireType = acquireWriteWordUncached - def getUncachedAtomicAcquireType = acquireAtomicUncached - def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck + def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = acquireReadExclusive def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = acquireReadExclusive @@ -251,80 +227,61 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) - case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) + case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } - def messageUpdatesDataArray (reply: Grant): Bool = { - (reply.g_type === grantReadExclusive) + def messageUpdatesDataArray(g: Grant): Bool = { + Mux(g.uncached, Bool(false), + (g.g_type === grantReadExclusive)) } - def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadExclusive -> grantReadExclusive, - acquireReadUncached -> grantReadUncached, - acquireWriteUncached -> grantWriteUncached, - acquireReadWordUncached -> grantReadWordUncached, - acquireWriteWordUncached -> grantWriteWordUncached, - acquireAtomicUncached -> grantAtomicUncached - )) - } + def getGrantType(a: Acquire, m: MasterMetadata): UInt = + Mux(a.uncached, getGrantTypeForUncached(a, m), grantReadExclusive) - def getGrantType(r: Release, m: MasterMetadata): UInt = { - MuxLookup(r.r_type, grantReadUncached, Array( - releaseVoluntaryInvalidateData -> grantVoluntaryAck - )) - } + def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck def getProbeType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, probeCopy, Array( - acquireReadExclusive -> probeInvalidate, - acquireReadUncached -> probeCopy, - acquireWriteUncached -> probeInvalidate, - acquireReadWordUncached -> probeCopy, - acquireWriteWordUncached -> probeInvalidate, - acquireAtomicUncached -> probeInvalidate - )) + Mux(a.uncached, + MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedRead -> probeCopy, + Acquire.uncachedWrite -> probeInvalidate, + Acquire.uncachedAtomic -> probeInvalidate + )), probeInvalidate) } - def requiresOuterRead(a_type: UInt) = { - (a_type != acquireWriteUncached) - } - def requiresOuterWrite(a_type: UInt) = { - (a_type === acquireWriteUncached) - } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) - def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck - def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + + def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck + def requiresAckForRelease(r: Release) = Bool(false) + def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 3 def nMasterStates = 2 - def nAcquireTypes = 7 + def nAcquireTypes = 2 def nProbeTypes = 3 def nReleaseTypes = 7 - def nGrantTypes = 8 + def nGrantTypes = 2 val clientInvalid :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) - val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) + val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) + val grantVoluntaryAck :: grantReadExclusive :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeVec = Vec(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) + val hasDataGrantTypeVec = Vec(grantReadExclusive) def isHit (cmd: UInt, m: ClientMetadata) = isValid(m) def isValid (m: ClientMetadata) = m.state != clientInvalid @@ -332,7 +289,7 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def isValid (m: MasterMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { - (isRead(cmd) && messageIsUncached(outstanding)) || + (isRead(cmd) && outstanding.uncached) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusiveDirty)) } def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { @@ -355,16 +312,9 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi )))(this) def clientMetadataOnFlush() = clientMetadataOnCacheControl(M_INV) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - MuxLookup(incoming.g_type, clientInvalid, Array( - grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusiveDirty, - clientExclusiveDirty, clientExclusiveClean), - grantReadExclusiveAck -> clientExclusiveDirty, - grantReadUncached -> clientInvalid, - grantWriteUncached -> clientInvalid, - grantReadWordUncached -> clientInvalid, - grantWriteWordUncached -> clientInvalid, - grantAtomicUncached -> clientInvalid - )))(this) + Mux(incoming.uncached, clientInvalid, + Mux(outstanding.a_type === acquireReadExclusiveDirty, clientExclusiveDirty, + clientExclusiveClean)))(this) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, @@ -387,14 +337,8 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi )) } - def getUncachedReadAcquireType = acquireReadUncached - def getUncachedWriteAcquireType = acquireWriteUncached - def getUncachedReadWordAcquireType = acquireReadWordUncached - def getUncachedWriteWordAcquireType = acquireWriteWordUncached - def getUncachedAtomicAcquireType = acquireAtomicUncached - def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck + def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, acquireReadExclusiveClean) @@ -419,83 +363,62 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) - case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) + case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } - def messageUpdatesDataArray (reply: Grant): Bool = { - (reply.g_type === grantReadExclusive) + def messageUpdatesDataArray(g: Grant): Bool = { + Mux(g.uncached, Bool(false), + (g.g_type === grantReadExclusive)) } - def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) def getGrantType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadExclusiveClean -> grantReadExclusive, - acquireReadExclusiveDirty -> grantReadExclusive, - acquireReadUncached -> grantReadUncached, - acquireWriteUncached -> grantWriteUncached, - acquireReadWordUncached -> grantReadWordUncached, - acquireWriteWordUncached -> grantWriteWordUncached, - acquireAtomicUncached -> grantAtomicUncached - )) + Mux(a.uncached, getGrantTypeForUncached(a, m), grantReadExclusive) } - def getGrantType(r: Release, m: MasterMetadata): UInt = { - MuxLookup(r.r_type, grantReadUncached, Array( - releaseVoluntaryInvalidateData -> grantVoluntaryAck - )) - } - + def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck def getProbeType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, probeCopy, Array( - acquireReadExclusiveClean -> probeInvalidate, - acquireReadExclusiveDirty -> probeInvalidate, - acquireReadUncached -> probeCopy, - acquireWriteUncached -> probeInvalidate, - acquireReadWordUncached -> probeCopy, - acquireWriteWordUncached -> probeInvalidate, - acquireAtomicUncached -> probeInvalidate - )) + Mux(a.uncached, + MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedRead -> probeCopy, + Acquire.uncachedWrite -> probeInvalidate, + Acquire.uncachedAtomic -> probeInvalidate + )), probeInvalidate) } - def requiresOuterRead(a_type: UInt) = { - (a_type != acquireWriteUncached) - } - def requiresOuterWrite(a_type: UInt) = { - (a_type === acquireWriteUncached) - } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) - def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck - def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + + def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck + def requiresAckForRelease(r: Release) = Bool(false) + def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 3 def nMasterStates = 3 - def nAcquireTypes = 7 + def nAcquireTypes = 2 def nProbeTypes = 3 def nReleaseTypes = 7 - def nGrantTypes = 9 + def nGrantTypes = 3 val clientInvalid :: clientShared :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) + val acquireReadShared :: acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) + val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive) def isHit (cmd: UInt, m: ClientMetadata): Bool = { Mux(isWriteIntent(cmd), (m.state === clientExclusiveDirty), @@ -508,7 +431,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def isValid (m: MasterMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { - (isRead(cmd) && messageIsUncached(outstanding)) || + (isRead(cmd) && outstanding.uncached) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) } def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { @@ -530,16 +453,9 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi )))(this) def clientMetadataOnFlush() = clientMetadataOnCacheControl(M_INV) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - MuxLookup(incoming.g_type, clientInvalid, Array( - grantReadShared -> clientShared, - grantReadExclusive -> clientExclusiveDirty, - grantReadExclusiveAck -> clientExclusiveDirty, - grantReadUncached -> clientInvalid, - grantWriteUncached -> clientInvalid, - grantReadWordUncached -> clientInvalid, - grantWriteWordUncached -> clientInvalid, - grantAtomicUncached -> clientInvalid - )))(this) + Mux(incoming.uncached, clientInvalid, + Mux(incoming.g_type === grantReadShared, clientShared, + clientExclusiveDirty)))(this) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, @@ -564,14 +480,8 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi )) } - def getUncachedReadAcquireType = acquireReadUncached - def getUncachedWriteAcquireType = acquireWriteUncached - def getUncachedReadWordAcquireType = acquireReadWordUncached - def getUncachedWriteWordAcquireType = acquireWriteWordUncached - def getUncachedAtomicAcquireType = acquireAtomicUncached - def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck + def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) @@ -596,79 +506,69 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) - case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) + case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } - def messageUpdatesDataArray (reply: Grant): Bool = { - (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) + def messageUpdatesDataArray(g: Grant): Bool = { + Mux(g.uncached, Bool(false), + (g.g_type === grantReadShared || g.g_type === grantReadExclusive)) } - def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) def getGrantType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), - acquireReadExclusive -> grantReadExclusive, - acquireReadUncached -> grantReadUncached, - acquireWriteUncached -> grantWriteUncached, - acquireReadWordUncached -> grantReadWordUncached, - acquireWriteWordUncached -> grantWriteWordUncached, - acquireAtomicUncached -> grantAtomicUncached - )) - } - def getGrantType(r: Release, m: MasterMetadata): UInt = { - MuxLookup(r.r_type, grantReadUncached, Array( - releaseVoluntaryInvalidateData -> grantVoluntaryAck - )) + Mux(a.uncached, getGrantTypeForUncached(a, m), + Mux(a.a_type === acquireReadShared, + Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), + grantReadExclusive)) } + def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck def getProbeType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, probeCopy, Array( - acquireReadShared -> probeDowngrade, - acquireReadExclusive -> probeInvalidate, - acquireReadUncached -> probeCopy, - acquireWriteUncached -> probeInvalidate - )) + Mux(a.uncached, + MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedRead -> probeCopy, + Acquire.uncachedWrite -> probeInvalidate, + Acquire.uncachedAtomic -> probeInvalidate + )), + MuxLookup(a.a_type, probeInvalidate, Array( + acquireReadShared -> probeDowngrade, + acquireReadExclusive -> probeInvalidate + ))) } - def requiresOuterRead(a_type: UInt) = { - (a_type != acquireWriteUncached) - } - def requiresOuterWrite(a_type: UInt) = { - (a_type === acquireWriteUncached) - } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) - def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck - def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + + def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck + def requiresAckForRelease(r: Release) = Bool(false) + def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 4 def nMasterStates = 3 - def nAcquireTypes = 7 + def nAcquireTypes = 2 def nProbeTypes = 3 def nReleaseTypes = 7 - def nGrantTypes = 9 + def nGrantTypes = 4 val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) + val acquireReadShared :: acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) + val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive) def isHit (cmd: UInt, m: ClientMetadata): Bool = { Mux(isWriteIntent(cmd), (m.state === clientExclusiveClean || m.state === clientExclusiveDirty), @@ -681,7 +581,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW def isValid (m: MasterMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { - (isRead(cmd) && messageIsUncached(outstanding)) || + (isRead(cmd) && outstanding.uncached) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) } def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { @@ -704,16 +604,13 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW )))(this) def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - MuxLookup(incoming.g_type, clientInvalid, Array( - grantReadShared -> clientShared, - grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusive, clientExclusiveDirty, clientExclusiveClean), - grantReadExclusiveAck -> clientExclusiveDirty, - grantReadUncached -> clientInvalid, - grantWriteUncached -> clientInvalid, - grantReadWordUncached -> clientInvalid, - grantWriteWordUncached -> clientInvalid, - grantAtomicUncached -> clientInvalid - )))(this) + Mux(incoming.uncached, clientInvalid, + MuxLookup(incoming.g_type, clientInvalid, Array( + grantReadShared -> clientShared, + grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusive, + clientExclusiveDirty, clientExclusiveClean), + grantReadExclusiveAck -> clientExclusiveDirty + ))))(this) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, @@ -738,14 +635,8 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW )) } - def getUncachedReadAcquireType = acquireReadUncached - def getUncachedWriteAcquireType = acquireWriteUncached - def getUncachedReadWordAcquireType = acquireReadWordUncached - def getUncachedWriteWordAcquireType = acquireWriteWordUncached - def getUncachedAtomicAcquireType = acquireAtomicUncached - def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck + def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) @@ -770,83 +661,68 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) - case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) + case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } - def messageUpdatesDataArray (reply: Grant): Bool = { - (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) + def messageUpdatesDataArray(g: Grant): Bool = { + Mux(g.uncached, Bool(false), + (g.g_type === grantReadShared || g.g_type === grantReadExclusive)) } - def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) def getGrantType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), - acquireReadExclusive -> grantReadExclusive, - acquireReadUncached -> grantReadUncached, - acquireWriteUncached -> grantWriteUncached, - acquireReadWordUncached -> grantReadWordUncached, - acquireWriteWordUncached -> grantWriteWordUncached, - acquireAtomicUncached -> grantAtomicUncached - )) + Mux(a.uncached, getGrantTypeForUncached(a, m), + Mux(a.a_type === acquireReadShared, + Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), + grantReadExclusive)) } - def getGrantType(r: Release, m: MasterMetadata): UInt = { - MuxLookup(r.r_type, grantReadUncached, Array( - releaseVoluntaryInvalidateData -> grantVoluntaryAck - )) - } - + def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck def getProbeType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, probeCopy, Array( - acquireReadShared -> probeDowngrade, - acquireReadExclusive -> probeInvalidate, - acquireReadUncached -> probeCopy, - acquireWriteUncached -> probeInvalidate, - acquireReadWordUncached -> probeCopy, - acquireWriteWordUncached -> probeInvalidate, - acquireAtomicUncached -> probeInvalidate - )) + Mux(a.uncached, + MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedRead -> probeCopy, + Acquire.uncachedWrite -> probeInvalidate, + Acquire.uncachedAtomic -> probeInvalidate + )), + MuxLookup(a.a_type, probeCopy, Array( + acquireReadShared -> probeDowngrade, + acquireReadExclusive -> probeInvalidate + ))) } - def requiresOuterRead(a_type: UInt) = { - (a_type != acquireWriteUncached) - } - def requiresOuterWrite(a_type: UInt) = { - (a_type === acquireWriteUncached) - } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) + def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) - def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck - def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck + def requiresAckForRelease(r: Release) = Bool(false) + def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 7 def nMasterStates = 3 - def nAcquireTypes = 8 + def nAcquireTypes = 3 def nProbeTypes = 4 def nReleaseTypes = 11 - def nGrantTypes = 9 + def nGrantTypes = 5 val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: clientSharedByTwo :: clientMigratoryClean :: clientMigratoryDirty :: Nil = Enum(UInt(), nClientStates) val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) + val acquireReadShared :: acquireReadExclusive :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(UInt(), nProbeTypes) val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(UInt(), nGrantTypes) + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadExclusiveAck :: grantReadMigratory :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadMigratory, grantReadWordUncached, grantAtomicUncached) + val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadMigratory) val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) def isHit (cmd: UInt, m: ClientMetadata): Bool = { @@ -859,7 +735,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def isValid (m: MasterMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { - (isRead(cmd) && messageIsUncached(outstanding)) || + (isRead(cmd) && outstanding.uncached) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive && outstanding.a_type != acquireInvalidateOthers)) } def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { @@ -883,22 +759,18 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo )))(this) def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - MuxLookup(incoming.g_type, clientInvalid, Array( - grantReadShared -> clientShared, - grantReadExclusive -> MuxLookup(outstanding.a_type, clientExclusiveDirty, Array( - acquireReadExclusive -> clientExclusiveDirty, - acquireReadShared -> clientExclusiveClean)), - grantReadExclusiveAck -> clientExclusiveDirty, - grantReadUncached -> clientInvalid, - grantWriteUncached -> clientInvalid, - grantReadWordUncached -> clientInvalid, - grantWriteWordUncached -> clientInvalid, - grantAtomicUncached -> clientInvalid, - grantReadMigratory -> MuxLookup(outstanding.a_type, clientMigratoryDirty, Array( - acquireInvalidateOthers -> clientMigratoryDirty, - acquireReadExclusive -> clientMigratoryDirty, - acquireReadShared -> clientMigratoryClean)) - )))(this) + Mux(incoming.uncached, clientInvalid, + MuxLookup(incoming.g_type, clientInvalid, Array( + grantReadShared -> clientShared, + grantReadExclusive -> MuxLookup(outstanding.a_type, clientExclusiveDirty, Array( + acquireReadExclusive -> clientExclusiveDirty, + acquireReadShared -> clientExclusiveClean)), + grantReadExclusiveAck -> clientExclusiveDirty, + grantReadMigratory -> MuxLookup(outstanding.a_type, clientMigratoryDirty, Array( + acquireInvalidateOthers -> clientMigratoryDirty, + acquireReadExclusive -> clientMigratoryDirty, + acquireReadShared -> clientMigratoryClean)) + ))))(this) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, @@ -935,14 +807,8 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo } - def getUncachedReadAcquireType = acquireReadUncached - def getUncachedWriteAcquireType = acquireWriteUncached - def getUncachedReadWordAcquireType = acquireReadWordUncached - def getUncachedWriteWordAcquireType = acquireWriteWordUncached - def getUncachedAtomicAcquireType = acquireAtomicUncached - def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck + def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { Mux(isWriteIntent(cmd), Mux(m.state === clientInvalid, acquireReadExclusive, acquireInvalidateOthers), acquireReadShared) @@ -969,60 +835,50 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) - case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) + case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } - def messageUpdatesDataArray (reply: Grant): Bool = Vec(grantReadShared, grantReadExclusive, grantReadMigratory).contains(reply.g_type) - def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) + def messageUpdatesDataArray(g: Grant): Bool = { + Mux(g.uncached, Bool(false), + Vec(grantReadShared, grantReadExclusive, grantReadMigratory).contains(g.g_type)) + } def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) def getGrantType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? - acquireReadExclusive -> grantReadExclusive, - acquireReadUncached -> grantReadUncached, - acquireWriteUncached -> grantWriteUncached, - acquireReadWordUncached -> grantReadWordUncached, - acquireWriteWordUncached -> grantWriteWordUncached, - acquireAtomicUncached -> grantAtomicUncached, - acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI? - )) + Mux(a.uncached, getGrantTypeForUncached(a, m), + MuxLookup(a.a_type, grantReadShared, Array( + acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), + acquireReadExclusive -> grantReadExclusive, + acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI for broadcast? + ))) } - def getGrantType(r: Release, m: MasterMetadata): UInt = { - MuxLookup(r.r_type, grantReadUncached, Array( - releaseVoluntaryInvalidateData -> grantVoluntaryAck - )) - } - + def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck def getProbeType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, probeCopy, Array( - acquireReadShared -> probeDowngrade, - acquireReadExclusive -> probeInvalidate, - acquireReadUncached -> probeCopy, - acquireWriteUncached -> probeInvalidate, - acquireReadWordUncached -> probeCopy, - acquireWriteWordUncached -> probeInvalidate, - acquireAtomicUncached -> probeInvalidate, - acquireInvalidateOthers -> probeInvalidateOthers - )) + Mux(a.uncached, + MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedRead -> probeCopy, + Acquire.uncachedWrite -> probeInvalidate, + Acquire.uncachedAtomic -> probeInvalidate + )), + MuxLookup(a.a_type, probeCopy, Array( + acquireReadShared -> probeDowngrade, + acquireReadExclusive -> probeInvalidate, + acquireInvalidateOthers -> probeInvalidateOthers + ))) } - def requiresOuterRead(a_type: UInt) = { - (a_type != acquireWriteUncached && a_type != acquireInvalidateOthers) - } - def requiresOuterWrite(a_type: UInt) = { - (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached || a_type === acquireAtomicUncached) - } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) + def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), acq.a_type != acquireInvalidateOthers) + def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) - def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck - def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck + def requiresAckForRelease(r: Release) = Bool(false) + def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index a402e7db..fe647d07 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -16,7 +16,9 @@ trait MemoryOpConstants { val MT_BU = Bits("b100") val MT_HU = Bits("b101") val MT_WU = Bits("b110") + val MT_CB = Bits("b111") // cache block + val NUM_XA_OPS = 9 val M_SZ = 5 val M_X = Bits("b?????"); val M_XRD = Bits("b00000"); // int load diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index edcd66cb..405380f2 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -135,7 +135,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { mem_acked := Bool(true) mem_gxid := io.mem.grant.bits.payload.master_xact_id mem_gsrc := io.mem.grant.bits.header.src - mem_needs_ack := co.requiresAckForGrant(io.mem.grant.bits.payload.g_type) + mem_needs_ack := co.requiresAckForGrant(io.mem.grant.bits.payload) } io.mem.grant.ready := Bool(true) @@ -193,8 +193,8 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { acq_q.io.enq.valid := state === state_mem_rreq || state === state_mem_wreq val init_addr = addr.toUInt >> UInt(offsetBits-3) acq_q.io.enq.bits := Mux(cmd === cmd_writemem, - Acquire(co.getUncachedWriteAcquireType, init_addr, UInt(0)), - Acquire(co.getUncachedReadAcquireType, init_addr, UInt(0))) + UncachedWrite(init_addr, UInt(0)), + UncachedRead(init_addr)) io.mem.acquire.valid := acq_q.io.deq.valid acq_q.io.deq.ready := io.mem.acquire.ready io.mem.acquire.bits.payload := acq_q.io.deq.bits diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index b470df49..02e651b8 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -268,7 +268,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { // Aggregate incoming MemIO responses into TL Grants io.mem.resp.ready := !active_in || cnt_in < UInt(cnt_max) io.uncached.grant.valid := active_in && (cnt_in === UInt(cnt_max)) - io.uncached.grant.bits.payload := Grant(UInt(0), tag_in, UInt(0), buf_in) + io.uncached.grant.bits.payload := Grant(Bool(true), UInt(0), tag_in, UInt(0), buf_in) when(!active_in && io.mem.resp.valid) { active_in := Bool(true) cnt_in := UInt(1) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 6738deaf..3b1bf26b 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -2,65 +2,89 @@ package uncore import Chisel._ +import scala.math.max case object TLId extends Field[String] -case object TLCoherence extends Field[CoherencePolicyWithUncached] +case object TLCoherence extends Field[CoherencePolicy] case object TLAddrBits extends Field[Int] case object TLMasterXactIdBits extends Field[Int] case object TLClientXactIdBits extends Field[Int] case object TLDataBits extends Field[Int] -case object TLWriteMaskBits extends Field[Int] -case object TLWordAddrBits extends Field[Int] -case object TLAtomicOpBits extends Field[Int] -trait HasPhysicalAddress extends Bundle { - val addr = UInt(width = params(TLAddrBits)) +abstract trait TileLinkParameters extends UsesParameters { + val tlAddrBits = params(TLAddrBits) + val tlClientXactIdBits = params(TLClientXactIdBits) + val tlMasterXactIdBits = params(TLMasterXactIdBits) + val tlDataBits = params(TLDataBits) + val tlWriteMaskBits = tlDataBits/8 + val tlSubblockAddrBits = log2Up(tlWriteMaskBits) + val tlAtomicOpcodeBits = log2Up(NUM_XA_OPS) + val tlUncachedOperandSizeBits = MT_SZ + val tlSubblockUnionBits = max(tlWriteMaskBits, + (tlSubblockAddrBits + + tlUncachedOperandSizeBits + + tlAtomicOpcodeBits)) } -trait HasClientTransactionId extends Bundle { - val client_xact_id = Bits(width = params(TLClientXactIdBits)) +class TLBundle extends Bundle with TileLinkParameters + +trait HasPhysicalAddress extends TLBundle { + val addr = UInt(width = tlAddrBits) } -trait HasMasterTransactionId extends Bundle { - val master_xact_id = Bits(width = params(TLMasterXactIdBits)) +trait HasClientTransactionId extends TLBundle { + val client_xact_id = Bits(width = tlClientXactIdBits) } -trait HasTileLinkData extends Bundle { - val data = Bits(width = params(TLDataBits)) +trait HasMasterTransactionId extends TLBundle { + val master_xact_id = Bits(width = tlMasterXactIdBits) } -trait SourcedMessage extends Bundle +trait HasTileLinkData extends TLBundle { + val data = UInt(width = tlDataBits) +} + +trait SourcedMessage extends TLBundle trait ClientSourcedMessage extends SourcedMessage trait MasterSourcedMessage extends SourcedMessage -object Acquire -{ - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt): Acquire = { +class Acquire extends ClientSourcedMessage + with HasPhysicalAddress + with HasClientTransactionId + with HasTileLinkData { + val uncached = Bool() + val a_type = UInt(width = max(log2Up(Acquire.nUncachedAcquireTypes), params(TLCoherence).acquireTypeWidth)) + val subblock = Bits(width = tlSubblockUnionBits) + val sbAddrOff = tlSubblockAddrBits + tlUncachedOperandSizeBits + val opSzOff = tlUncachedOperandSizeBits + sbAddrOff + def operand_sz(dummy: Int = 0) = subblock(tlUncachedOperandSizeBits-1, 0) + def subblock_addr(dummy: Int = 0) = subblock(sbAddrOff-1, tlUncachedOperandSizeBits) + def atomic_op(dummy: Int = 0) = subblock(opSzOff-1, sbAddrOff) + def write_mask(dummy: Int = 0) = subblock(tlWriteMaskBits-1, 0) +} + +object Acquire { + val nUncachedAcquireTypes = 3 + //val uncachedRead :: uncachedWrite :: uncachedAtomic :: Nil = Enum(UInt(), nUncachedAcquireTypes) + def uncachedRead = UInt(0) + def uncachedWrite = UInt(1) + def uncachedAtomic = UInt(2) + def hasData(a_type: UInt) = Vec(uncachedWrite, uncachedAtomic).contains(a_type) + def requiresOuterRead(a_type: UInt) = a_type != uncachedWrite + def requiresOuterWrite(a_type: UInt) = a_type === uncachedWrite + + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt): Acquire = { val acq = new Acquire + acq.uncached := Bool(false) acq.a_type := a_type acq.addr := addr acq.client_xact_id := client_xact_id - acq.data := Bits(0) - acq.write_mask := Bits(0) - acq.subword_addr := Bits(0) - acq.atomic_opcode := Bits(0) - acq - } - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt): Acquire = { - val acq = apply(a_type, addr, client_xact_id) acq.data := data + acq.subblock := UInt(0) acq } - def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, write_mask: Bits, data: UInt): Acquire = { - val acq = apply(a_type, addr, client_xact_id, data) - acq.write_mask := write_mask - acq - } - def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt, data: UInt): Acquire = { - val acq = apply(a_type, addr, client_xact_id, data) - acq.subword_addr := subword_addr - acq.atomic_opcode := atomic_opcode - acq + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt): Acquire = { + apply(a_type, addr, client_xact_id, UInt(0)) } def apply(a: Acquire): Acquire = { val acq = new Acquire @@ -69,16 +93,45 @@ object Acquire } } -class Acquire extends ClientSourcedMessage - with HasPhysicalAddress - with HasClientTransactionId - with HasTileLinkData { - val a_type = UInt(width = params(TLCoherence).acquireTypeWidth) - val write_mask = Bits(width = params(TLWriteMaskBits)) - val subword_addr = Bits(width = params(TLWordAddrBits)) - val atomic_opcode = Bits(width = params(TLAtomicOpBits)) +object UncachedRead { + def apply(addr: UInt, client_xact_id: UInt, subblock_addr: UInt, operand_sz: UInt): Acquire = { + val acq = Acquire(Acquire.uncachedRead, addr, client_xact_id) + acq.uncached := Bool(true) + acq.subblock := Cat(subblock_addr, operand_sz) + acq + } + def apply(addr: UInt, client_xact_id: UInt): Acquire = { + apply(addr, client_xact_id, UInt(0), MT_CB) + } + def apply(addr: UInt): Acquire = { + apply(addr, UInt(0), UInt(0), MT_CB) + } } +object UncachedWrite { + def apply(addr: UInt, client_xact_id: UInt, write_mask: Bits, data: UInt): Acquire = { + val acq = Acquire(Acquire.uncachedWrite, addr, client_xact_id, data) + acq.uncached := Bool(true) + acq.subblock := write_mask + acq + } + def apply(addr: UInt, client_xact_id: UInt, data: UInt): Acquire = { + apply(addr, client_xact_id, SInt(-1), data) + } + def apply(addr: UInt, data: UInt): Acquire = { + apply(addr, UInt(0), data) + } +} + +object UncachedAtomic { + def apply(addr: UInt, client_xact_id: UInt, atomic_opcode: UInt, + subblock_addr: UInt, operand_sz: UInt, data: UInt): Acquire = { + val acq = Acquire(Acquire.uncachedAtomic, addr, client_xact_id, data) + acq.uncached := Bool(true) + acq.subblock := Cat(atomic_opcode, subblock_addr, operand_sz) + acq + } +} object Probe { def apply(p_type: UInt, addr: UInt) = { @@ -118,28 +171,34 @@ class Release extends ClientSourcedMessage val r_type = UInt(width = params(TLCoherence).releaseTypeWidth) } -object Grant -{ - def apply(g_type: UInt, client_xact_id: UInt, master_xact_id: UInt): Grant = { - val gnt = new Grant - gnt.g_type := g_type - gnt.client_xact_id := client_xact_id - gnt.master_xact_id := master_xact_id - gnt.data := UInt(0) - gnt - } - def apply(g_type: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt): Grant = { - val gnt = apply(g_type, client_xact_id, master_xact_id) - gnt.data := data - gnt - } -} - class Grant extends MasterSourcedMessage with HasTileLinkData with HasClientTransactionId with HasMasterTransactionId { - val g_type = UInt(width = params(TLCoherence).grantTypeWidth) + val uncached = Bool() + val g_type = UInt(width = max(log2Up(Grant.nUncachedGrantTypes), params(TLCoherence).grantTypeWidth)) +} + +object Grant { + val nUncachedGrantTypes = 3 + //val uncachedRead :: uncachedWrite :: uncachedAtomic :: Nil = Enum(UInt(), nUncachedGrantTypes) + def uncachedRead = UInt(0) + def uncachedWrite = UInt(1) + def uncachedAtomic = UInt(2) + def hasData(g_type: UInt) = Vec(uncachedRead, uncachedAtomic).contains(g_type) + + def apply(uncached: Bool, g_type: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt): Grant = { + val gnt = new Grant + gnt.uncached := uncached + gnt.g_type := g_type + gnt.client_xact_id := client_xact_id + gnt.master_xact_id := master_xact_id + gnt.data := data + gnt + } + def apply(uncached: Bool, g_type: UInt, client_xact_id: UInt, master_xact_id: UInt): Grant = { + apply(uncached, g_type, client_xact_id, master_xact_id, UInt(0)) + } } class Finish extends ClientSourcedMessage with HasMasterTransactionId diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index e1f59e08..6ea8cd25 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -122,7 +122,7 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.header.src := UInt(bankId) //io.outer.acquire.bits.header.dst TODO - io.outer.acquire.bits.payload := Bundle(Acquire(co.getUncachedWriteAcquireType, + io.outer.acquire.bits.payload := Bundle(UncachedWrite( xact.addr, UInt(trackerId), xact.data), @@ -133,7 +133,8 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id - io.inner.grant.bits.payload := Grant(co.getGrantType(xact, co.masterMetadataOnFlush), + io.inner.grant.bits.payload := Grant(Bool(false), + co.getGrantTypeOnVoluntaryWriteback(co.masterMetadataOnFlush), xact.client_xact_id, UInt(trackerId)) @@ -169,15 +170,13 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val curr_p_id = PriorityEncoder(probe_flags) val pending_outer_write = co.messageHasData(xact) - val pending_outer_read = co.requiresOuterRead(xact.a_type) - val outer_write_acq = Bundle(Acquire(co.getUncachedWriteAcquireType, - xact.addr, UInt(trackerId), xact.data), - { case TLId => outerId }) - val outer_write_rel = Bundle(Acquire(co.getUncachedWriteAcquireType, - xact.addr, UInt(trackerId), c_rel.payload.data), - { case TLId => outerId }) - val outer_read = Bundle(Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)), - { case TLId => outerId }) + val pending_outer_read = co.requiresOuterRead(xact, co.masterMetadataOnFlush) + val outer_write_acq = Bundle(UncachedWrite(xact.addr, UInt(trackerId), xact.data), + { case TLId => outerId }) + val outer_write_rel = Bundle(UncachedWrite(xact.addr, UInt(trackerId), c_rel.payload.data), + { case TLId => outerId }) + val outer_read = Bundle(UncachedRead(xact.addr, UInt(trackerId)), + { case TLId => outerId }) val probe_initial_flags = Bits(width = nClients) probe_initial_flags := Bits(0) @@ -202,11 +201,11 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri io.inner.probe.bits.header.dst := curr_p_id io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.masterMetadataOnFlush), xact.addr) - val grant_type = co.getGrantType(xact, co.masterMetadataOnFlush) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id - io.inner.grant.bits.payload := Grant(grant_type, + io.inner.grant.bits.payload := Grant(xact.uncached, + co.getGrantType(xact, co.masterMetadataOnFlush), xact.client_xact_id, UInt(trackerId), m_gnt.payload.data) @@ -218,7 +217,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri is(s_idle) { io.inner.acquire.ready := Bool(true) val needs_outer_write = co.messageHasData(c_acq.payload) - val needs_outer_read = co.requiresOuterRead(c_acq.payload.a_type) + val needs_outer_read = co.requiresOuterRead(c_acq.payload, co.masterMetadataOnFlush) when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src @@ -266,7 +265,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri io.outer.acquire.valid := Bool(true) io.outer.acquire.bits.payload := outer_read when(io.outer.acquire.ready) { - state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) + state := Mux(co.requiresAckForGrant(io.inner.grant.bits.payload), s_busy, s_idle) } } is(s_mem_write) { @@ -279,7 +278,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri is(s_make_grant) { io.inner.grant.valid := Bool(true) when(io.inner.grant.ready) { - state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) + state := Mux(co.requiresAckForGrant(io.inner.grant.bits.payload), s_busy, s_idle) } } is(s_busy) { // Nothing left to do but wait for transaction to complete From cb7e7125993c001c08d5159aa1e4d8bda3837c00 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 12 Nov 2014 12:55:07 -0800 Subject: [PATCH 261/374] Added uncached write data queue to coherence hub --- uncore/src/main/scala/tilelink.scala | 2 +- uncore/src/main/scala/uncore.scala | 59 +++++++++++++++++++++------- 2 files changed, 46 insertions(+), 15 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 3b1bf26b..ec4c15df 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -16,7 +16,7 @@ abstract trait TileLinkParameters extends UsesParameters { val tlClientXactIdBits = params(TLClientXactIdBits) val tlMasterXactIdBits = params(TLMasterXactIdBits) val tlDataBits = params(TLDataBits) - val tlWriteMaskBits = tlDataBits/8 + val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits val tlSubblockAddrBits = log2Up(tlWriteMaskBits) val tlAtomicOpcodeBits = log2Up(NUM_XA_OPS) val tlUncachedOperandSizeBits = MT_SZ diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 6ea8cd25..858d603c 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -5,14 +5,17 @@ import Chisel._ case object NReleaseTransactors extends Field[Int] case object NAcquireTransactors extends Field[Int] +case object L2StoreDataQueueDepth extends Field[Int] case object NClients extends Field[Int] abstract trait CoherenceAgentParameters extends UsesParameters { val co = params(TLCoherence) - val nReleaseTransactors = params(NReleaseTransactors) + val nReleaseTransactors = 1 val nAcquireTransactors = params(NAcquireTransactors) val nTransactors = nReleaseTransactors + nAcquireTransactors val nClients = params(NClients) + val sdqDepth = params(L2StoreDataQueueDepth) + val sdqIdBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(params(L2StoreDataQueueDepth))) + 1 } abstract class CoherenceAgent(innerId: String, outerId: String) extends Module @@ -27,11 +30,19 @@ abstract class CoherenceAgent(innerId: String, outerId: String) extends Module class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends CoherenceAgent(innerId, outerId) { + // Queue to store impending UncachedWrite data + val sdq_val = Reg(init=Bits(0, sdqDepth)) + val sdq_alloc_id = PriorityEncoder(~sdq_val(sdqDepth-1,0)) + val sdq_rdy = !sdq_val.andR + val sdq_enq = io.inner.acquire.valid && io.inner.acquire.ready && co.messageHasData(io.inner.acquire.bits.payload) + val sdq = Vec.fill(sdqDepth){Reg(io.inner.acquire.bits.payload.data)} + when (sdq_enq) { sdq(sdq_alloc_id) := io.inner.acquire.bits.payload.data } + // Create SHRs for outstanding transactions val trackerList = (0 until nReleaseTransactors).map(id => - Module(new VoluntaryReleaseTracker(id, bankId, innerId, outerId))) ++ + Module(new VoluntaryReleaseTracker(id, bankId, innerId, outerId), {case TLDataBits => sdqIdBits})) ++ (nReleaseTransactors until nTransactors).map(id => - Module(new AcquireTracker(id, bankId, innerId, outerId))) + Module(new AcquireTracker(id, bankId, innerId, outerId), {case TLDataBits => sdqIdBits})) // Propagate incoherence flags trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) @@ -46,10 +57,11 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends val t = trackerList(i).io.inner alloc_arb.io.in(i).valid := t.acquire.ready t.acquire.bits := acquire.bits + t.acquire.bits.payload.data := Cat(sdq_alloc_id, UInt(1)) t.acquire.valid := alloc_arb.io.in(i).ready } - acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && !block_acquires - alloc_arb.io.out.ready := acquire.valid && !block_acquires + acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && sdq_rdy && !block_acquires + alloc_arb.io.out.ready := acquire.valid && sdq_rdy && !block_acquires // Handle probe request generation val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) @@ -67,12 +79,19 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.inner t.release.bits := release.bits + t.release.bits.payload.data := (if (i < nReleaseTransactors) Cat(UInt(i), UInt(2)) else UInt(0)) t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases } release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases + val vwbdq = Vec.fill(nReleaseTransactors){ Reg(release.bits.payload.data) } + when(voluntary && release.fire()) { + vwbdq(release_idx) := release.bits.payload.data + } + // Reply to initial requestor val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) + io.inner.grant.bits.payload.data := io.outer.grant.bits.payload.data io.inner.grant <> grant_arb.io.out grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } @@ -84,9 +103,22 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends // Create an arbiter for the one memory port val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), - {case TLId => outerId}) + { case TLId => outerId; case TLDataBits => sdqIdBits }) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } + val is_in_sdq = outer_arb.io.out.acquire.bits.payload.data(0) + val is_in_vwbdq = outer_arb.io.out.acquire.bits.payload.data(1) + val free_sdq_id = outer_arb.io.out.acquire.bits.payload.data >> UInt(1) + val free_vwbdq_id = outer_arb.io.out.acquire.bits.payload.data >> UInt(2) + val free_sdq = io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload) && is_in_sdq + io.outer.acquire.bits.payload.data := Mux(is_in_sdq, sdq(free_sdq_id), + Mux(is_in_vwbdq, vwbdq(free_vwbdq_id), release.bits.payload.data)) io.outer <> outer_arb.io.out + + // Update SDQ valid bits + when (io.outer.acquire.valid || sdq_enq) { + sdq_val := sdq_val & ~(UIntToOH(free_sdq_id) & Fill(sdqDepth, free_sdq)) | + PriorityEncoderOH(~sdq_val(sdqDepth-1,0)) & Fill(sdqDepth, sdq_enq) + } } @@ -112,10 +144,9 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute val state = Reg(init=s_idle) val xact = Reg{ new Release } val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) - val incoming_rel = io.inner.release.bits io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, incoming_rel.payload.addr) && + io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) io.outer.grant.ready := Bool(false) @@ -140,11 +171,11 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute switch (state) { is(s_idle) { - io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { - xact := incoming_rel.payload - init_client_id := incoming_rel.header.src - state := Mux(co.messageHasData(incoming_rel.payload), s_mem, s_ack) + io.inner.release.ready := Bool(true) + xact := c_rel.payload + init_client_id := c_rel.header.src + state := Mux(co.messageHasData(c_rel.payload), s_mem, s_ack) } } is(s_mem) { @@ -173,7 +204,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val pending_outer_read = co.requiresOuterRead(xact, co.masterMetadataOnFlush) val outer_write_acq = Bundle(UncachedWrite(xact.addr, UInt(trackerId), xact.data), { case TLId => outerId }) - val outer_write_rel = Bundle(UncachedWrite(xact.addr, UInt(trackerId), c_rel.payload.data), + val outer_write_rel = Bundle(UncachedWrite(xact.addr, UInt(trackerId), UInt(0)), // Special SQDId { case TLId => outerId }) val outer_read = Bundle(UncachedRead(xact.addr, UInt(trackerId)), { case TLId => outerId }) @@ -208,7 +239,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri co.getGrantType(xact, co.masterMetadataOnFlush), xact.client_xact_id, UInt(trackerId), - m_gnt.payload.data) + UInt(0)) // Data bypassed in parent io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) From 05b5188ad97607b5534b60a56513acc8d853fcea Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 19 Nov 2014 15:55:25 -0800 Subject: [PATCH 262/374] meta and data bundle refactor --- uncore/src/main/scala/cache.scala | 198 +++++++++++++++--------------- 1 file changed, 100 insertions(+), 98 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 10d00210..ee375784 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -138,12 +138,19 @@ class L2MetaResp extends L2HellaCacheBundle with HasL2Id with HasL2InternalRequestState +trait HasL2MetaReadIO extends L2HellaCacheBundle { + val read = Decoupled(new L2MetaReadReq) + val resp = Valid(new L2MetaResp).flip +} + +trait HasL2MetaWriteIO extends L2HellaCacheBundle { + val write = Decoupled(new L2MetaWriteReq) +} + +class L2MetaRWIO extends L2HellaCacheBundle with HasL2MetaReadIO with HasL2MetaWriteIO + class L2MetadataArray extends L2HellaCacheModule { - val io = new Bundle { - val read = Decoupled(new L2MetaReadReq).flip - val write = Decoupled(new L2MetaWriteReq).flip - val resp = Valid(new L2MetaResp) - } + val io = new L2MetaRWIO().flip val meta = Module(new MetadataArray(() => L2Metadata(UInt(0), co.masterMetadataOnFlush))) meta.io.read <> io.read @@ -189,12 +196,19 @@ class L2DataResp extends Bundle with HasL2Id with TileLinkParameters { val data = Bits(width = tlDataBits) } +trait HasL2DataReadIO extends L2HellaCacheBundle { + val read = Decoupled(new L2DataReadReq) + val resp = Valid(new L2DataResp).flip +} + +trait HasL2DataWriteIO extends L2HellaCacheBundle { + val write = Decoupled(new L2DataWriteReq) +} + +class L2DataRWIO extends L2HellaCacheBundle with HasL2DataReadIO with HasL2DataWriteIO + class L2DataArray extends L2HellaCacheModule { - val io = new Bundle { - val read = Decoupled(new L2DataReadReq).flip - val write = Decoupled(new L2DataWriteReq).flip - val resp = Valid(new L2DataResp) - } + val io = new L2DataRWIO().flip val waddr = io.write.bits.addr val raddr = io.read.bits.addr @@ -226,12 +240,8 @@ class L2HellaCache(bankId: Int, innerId: String, outerId: String) extends val data = Module(new L2DataArray) tshrfile.io.inner <> io.inner - tshrfile.io.meta_read <> meta.io.read - tshrfile.io.meta_write <> meta.io.write - tshrfile.io.meta_resp <> meta.io.resp - tshrfile.io.data_read <> data.io.read - tshrfile.io.data_write <> data.io.write - tshrfile.io.data_resp <> data.io.resp + tshrfile.io.meta <> meta.io + tshrfile.io.data <> data.io io.outer <> tshrfile.io.outer io.incoherent <> tshrfile.io.incoherent } @@ -242,12 +252,8 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) val incoherent = Vec.fill(nClients){Bool()}.asInput - val meta_read = Decoupled(new L2MetaReadReq) - val meta_write = Decoupled(new L2MetaWriteReq) - val meta_resp = Valid(new L2MetaResp).flip - val data_read = Decoupled(new L2DataReadReq) - val data_write = Decoupled(new L2DataWriteReq) - val data_resp = Valid(new L2DataResp).flip + val meta = new L2MetaRWIO + val data = new L2DataRWIO } // Wiring helper funcs @@ -323,12 +329,12 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac io.outer <> outer_arb.io.out // Local memory - doOutputArbitration(io.meta_read, trackerList.map(_.io.meta_read)) - doOutputArbitration(io.meta_write, trackerList.map(_.io.meta_write)) - doOutputArbitration(io.data_read, trackerList.map(_.io.data_read)) - doOutputArbitration(io.data_write, trackerList.map(_.io.data_write)) - doInputRouting(io.meta_resp, trackerList.map(_.io.meta_resp)) - doInputRouting(io.data_resp, trackerList.map(_.io.data_resp)) + doOutputArbitration(io.meta.read, trackerList.map(_.io.meta.read)) + doOutputArbitration(io.meta.write, trackerList.map(_.io.meta.write)) + doOutputArbitration(io.data.read, trackerList.map(_.io.data.read)) + doOutputArbitration(io.data.write, trackerList.map(_.io.data.write)) + doInputRouting(io.meta.resp, trackerList.map(_.io.meta.resp)) + doInputRouting(io.data.resp, trackerList.map(_.io.data.resp)) } @@ -340,12 +346,8 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa val tile_incoherent = Bits(INPUT, nClients) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) - val meta_read = Decoupled(new L2MetaReadReq) - val meta_write = Decoupled(new L2MetaWriteReq) - val meta_resp = Valid(new L2MetaResp).flip - val data_read = Decoupled(new L2DataReadReq) - val data_write = Decoupled(new L2DataWriteReq) - val data_resp = Valid(new L2DataResp).flip + val data = new L2DataRWIO + val meta = new L2MetaRWIO } val c_acq = io.inner.acquire.bits @@ -386,23 +388,23 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou xact.client_xact_id, UInt(trackerId)) - io.data_read.valid := Bool(false) - io.data_write.valid := Bool(false) - io.data_write.bits.id := UInt(trackerId) - io.data_write.bits.way_en := xact_internal.way_en - io.data_write.bits.addr := xact.addr - io.data_write.bits.wmask := SInt(-1) - io.data_write.bits.data := xact.data - io.meta_read.valid := Bool(false) - io.meta_read.bits.id := UInt(trackerId) - io.meta_read.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta_read.bits.tag := xact.addr >> UInt(untagBits) - io.meta_write.valid := Bool(false) - io.meta_write.bits.id := UInt(trackerId) - io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta_write.bits.way_en := xact_internal.way_en - io.meta_write.bits.data.tag := xact.addr >> UInt(untagBits) - io.meta_write.bits.data.coh := co.masterMetadataOnRelease(xact, + io.data.read.valid := Bool(false) + io.data.write.valid := Bool(false) + io.data.write.bits.id := UInt(trackerId) + io.data.write.bits.way_en := xact_internal.way_en + io.data.write.bits.addr := xact.addr + io.data.write.bits.wmask := SInt(-1) + io.data.write.bits.data := xact.data + io.meta.read.valid := Bool(false) + io.meta.read.bits.id := UInt(trackerId) + io.meta.read.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta.read.bits.tag := xact.addr >> UInt(untagBits) + io.meta.write.valid := Bool(false) + io.meta.write.bits.id := UInt(trackerId) + io.meta.write.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta.write.bits.way_en := xact_internal.way_en + io.meta.write.bits.data.tag := xact.addr >> UInt(untagBits) + io.meta.write.bits.data.coh := co.masterMetadataOnRelease(xact, xact_internal.meta.coh, init_client_id) @@ -416,24 +418,24 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou } } is(s_meta_read) { - io.meta_read.valid := Bool(true) - when(io.meta_read.ready) { state := s_meta_resp } + io.meta.read.valid := Bool(true) + when(io.meta.read.ready) { state := s_meta_resp } } is(s_meta_resp) { - when(io.meta_resp.valid) { - xact_internal := io.meta_resp.bits - state := Mux(io.meta_resp.bits.tag_match, + when(io.meta.resp.valid) { + xact_internal := io.meta.resp.bits + state := Mux(io.meta.resp.bits.tag_match, Mux(co.messageHasData(xact), s_data_write, s_meta_write), s_grant) } } is(s_data_write) { - io.data_write.valid := Bool(true) - when(io.data_write.ready) { state := s_meta_write } + io.data.write.valid := Bool(true) + when(io.data.write.ready) { state := s_meta_write } } is(s_meta_write) { - io.meta_write.valid := Bool(true) - when(io.meta_write.ready) { state := s_grant } + io.meta.write.valid := Bool(true) + when(io.meta.write.ready) { state := s_grant } } is(s_grant) { io.inner.grant.valid := Bool(true) @@ -522,26 +524,26 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.inner.release.ready := Bool(false) io.inner.finish.ready := Bool(false) - io.data_read.valid := Bool(false) - io.data_read.bits.id := UInt(trackerId) - io.data_read.bits.way_en := xact_internal.way_en - io.data_read.bits.addr := xact.addr - io.data_write.valid := Bool(false) - io.data_write.bits.id := UInt(trackerId) - io.data_write.bits.way_en := xact_internal.way_en - io.data_write.bits.addr := xact.addr - io.data_write.bits.wmask := SInt(-1) - io.data_write.bits.data := xact.data - io.meta_read.valid := Bool(false) - io.meta_read.bits.id := UInt(trackerId) - io.meta_read.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta_read.bits.tag := xact.addr >> UInt(untagBits) - io.meta_write.valid := Bool(false) - io.meta_write.bits.id := UInt(trackerId) - io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta_write.bits.way_en := xact_internal.way_en - io.meta_write.bits.data.tag := xact.addr >> UInt(untagBits) - io.meta_write.bits.data.coh := next_coh_on_grant + io.data.read.valid := Bool(false) + io.data.read.bits.id := UInt(trackerId) + io.data.read.bits.way_en := xact_internal.way_en + io.data.read.bits.addr := xact.addr + io.data.write.valid := Bool(false) + io.data.write.bits.id := UInt(trackerId) + io.data.write.bits.way_en := xact_internal.way_en + io.data.write.bits.addr := xact.addr + io.data.write.bits.wmask := SInt(-1) + io.data.write.bits.data := xact.data + io.meta.read.valid := Bool(false) + io.meta.read.bits.id := UInt(trackerId) + io.meta.read.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta.read.bits.tag := xact.addr >> UInt(untagBits) + io.meta.write.valid := Bool(false) + io.meta.write.bits.id := UInt(trackerId) + io.meta.write.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta.write.bits.way_en := xact_internal.way_en + io.meta.write.bits.data.tag := xact.addr >> UInt(untagBits) + io.meta.write.bits.data.coh := next_coh_on_grant switch (state) { is(s_idle) { @@ -553,17 +555,17 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } } is(s_meta_read) { - io.meta_read.valid := Bool(true) - when(io.meta_read.ready) { state := s_meta_resp } + io.meta.read.valid := Bool(true) + when(io.meta.read.ready) { state := s_meta_resp } } is(s_meta_resp) { - when(io.meta_resp.valid) { - val coh = io.meta_resp.bits.meta.coh - val _tag_match = io.meta_resp.bits.tag_match + when(io.meta.resp.valid) { + val coh = io.meta.resp.bits.meta.coh + val _tag_match = io.meta.resp.bits.tag_match val _needs_writeback = !_tag_match && co.needsWriteback(coh) val _is_hit = _tag_match && co.isHit(xact, coh) val _needs_probes = co.requiresProbes(xact, coh) - xact_internal := io.meta_resp.bits + xact_internal := io.meta.resp.bits when(_needs_probes) { val mask_incoherent = co.dir().full(coh.sharers) & ~io.tile_incoherent val mask_self = mask_incoherent & @@ -626,14 +628,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } } is(s_data_read_wb) { - io.data_read.valid := Bool(true) - when(io.data_read.ready) { + io.data.read.valid := Bool(true) + when(io.data.read.ready) { state := s_data_resp_wb } } is(s_data_resp_wb) { - when(io.data_resp.valid) { - wb_buffer := io.data_resp.bits.data + when(io.data.resp.valid) { + wb_buffer := io.data.resp.bits.data state := s_outer_write_wb } } @@ -661,26 +663,26 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } } is(s_data_read_hit) { - io.data_read.valid := Bool(true) - when(io.data_read.ready) { + io.data.read.valid := Bool(true) + when(io.data.read.ready) { state := s_data_resp_hit } } is(s_data_resp_hit) { - when(io.data_resp.valid) { - xact.data := mergeData(xact, io.data_resp.bits.data) + when(io.data.resp.valid) { + xact.data := mergeData(xact, io.data.resp.bits.data) state := s_meta_write } } is(s_data_write) { - io.data_write.valid := Bool(true) - when(io.data_write.ready) { + io.data.write.valid := Bool(true) + when(io.data.write.ready) { state := s_meta_write } } is(s_meta_write) { - io.meta_write.valid := Bool(true) - when(io.meta_write.ready) { state := s_grant } + io.meta.write.valid := Bool(true) + when(io.meta.write.ready) { state := s_grant } } is(s_grant) { io.inner.grant.valid := Bool(true) From 404773eb9f1f29277b205eb4ace622ca67eb0d9d Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 3 Dec 2014 14:22:39 -0800 Subject: [PATCH 263/374] fix wb bug --- uncore/src/main/scala/cache.scala | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index ee375784..0a717604 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -462,6 +462,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val crel_wb_src = Reg(init = UInt(0, width = log2Up(nClients))) val crel_wb_g_type = Reg(init = UInt(0, width = co.grantTypeWidth)) val wb_buffer = Reg{xact.data.clone} + val addr_wb = Cat(xact_internal.meta.tag, xact.addr(untagBits-1,blockOffBits)) val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) val pending_probes = Reg(init = co.dir().flush) @@ -476,8 +477,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) //TODO: Also indexes - io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && - (state != s_idle) //TODO: Also indexes? + io.has_release_conflict := (co.isCoherenceConflict(xact.addr, c_rel.payload.addr) || + co.isCoherenceConflict(addr_wb, c_rel.payload.addr)) && + (state != s_idle) //TODO: Also indexes? val next_coh_on_release = co.masterMetadataOnRelease( c_rel.payload, @@ -488,7 +490,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St xact_internal.meta.coh, c_gnt.header.dst) - val addr_wb = Cat(xact_internal.meta.tag, xact.addr(untagBits-1,blockOffBits)) val outer_write_acq = Bundle(UncachedWrite(xact.addr, UInt(trackerId), xact.data), { case TLId => outerId }) val outer_write_wb = Bundle(UncachedWrite(addr_wb, UInt(trackerId), wb_buffer), From 2f733a60db1d564f2da11632acc6e3308910460c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 7 Dec 2014 02:57:44 -0800 Subject: [PATCH 264/374] Begin adding TLDataBeats to uncore --- uncore/src/main/scala/network.scala | 4 ++-- uncore/src/main/scala/tilelink.scala | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index d3f4a9e8..a34e91c1 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -25,13 +25,13 @@ class BasicCrossbarIO[T <: Data](n: Int, dType: T) extends Bundle { abstract class PhysicalNetwork extends Module -class BasicCrossbar[T <: Data](n: Int, dType: T, count: Int = 1) extends PhysicalNetwork { +class BasicCrossbar[T <: Data](n: Int, dType: T, count: Int = 1, needsLock: Option[PhysicalNetworkIO[T] => Bool] = None) extends PhysicalNetwork { val io = new BasicCrossbarIO(n, dType) val rdyVecs = List.fill(n){Vec.fill(n)(Bool())} io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { - val rrarb = Module(new LockingRRArbiter(io.in(0).bits, n, count)) + val rrarb = Module(new LockingRRArbiter(io.in(0).bits, n, count, needsLock)) (rrarb.io.in, io.in, rdys).zipped.map{ case (arb, in, rdy) => { arb.valid := in.valid && (in.bits.header.dst === UInt(i)) arb.bits := in.bits diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index a5066f39..1e535507 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -10,12 +10,14 @@ case object TLAddrBits extends Field[Int] case object TLMasterXactIdBits extends Field[Int] case object TLClientXactIdBits extends Field[Int] case object TLDataBits extends Field[Int] +case object TLDataBeats extends Field[Int] abstract trait TileLinkParameters extends UsesParameters { val tlAddrBits = params(TLAddrBits) val tlClientXactIdBits = params(TLClientXactIdBits) val tlMasterXactIdBits = params(TLMasterXactIdBits) val tlDataBits = params(TLDataBits) + val tlDataBeats = params(TLDataBeats) val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits val tlSubblockAddrBits = log2Up(tlWriteMaskBits) val tlAtomicOpcodeBits = log2Up(NUM_XA_OPS) From 3026c46a9c81709bd1c49e1f9737af1095cf918c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 7 Dec 2014 03:02:20 -0800 Subject: [PATCH 265/374] Finish adding TLDataBeats to uncore & hub --- uncore/src/main/scala/cache.scala | 4 +- uncore/src/main/scala/coherence.scala | 6 - uncore/src/main/scala/htif.scala | 33 +++-- uncore/src/main/scala/memserdes.scala | 133 ++++++++++++------ uncore/src/main/scala/tilelink.scala | 18 +-- uncore/src/main/scala/uncore.scala | 193 ++++++++++++++++---------- uncore/src/main/scala/util.scala | 88 ++++++++++++ 7 files changed, 325 insertions(+), 150 deletions(-) create mode 100644 uncore/src/main/scala/util.scala diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 0a717604..b96aab70 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -30,7 +30,8 @@ abstract trait CacheParameters extends UsesParameters { val rowWords = rowBits/wordBits val rowBytes = rowBits/8 val rowOffBits = log2Up(rowBytes) - val refillCycles = params(TLDataBits)/rowBits + val refillCyclesPerBeat = params(TLDataBits)/rowBits + val refillCycles = refillCyclesPerBeat*params(TLDataBeats) } abstract class CacheBundle extends Bundle with CacheParameters @@ -99,7 +100,6 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgentParameters - with TileLinkParameters abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 57545fda..280e20ad 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -3,12 +3,6 @@ package uncore import Chisel._ -object MuxBundle { - def apply[T <: Data] (default: T, mapping: Seq[(Bool, T)]): T = { - mapping.reverse.foldLeft(default)((b, a) => Mux(a._1, a._2, b)) - } -} - abstract class CoherenceMetadata extends Bundle object ClientMetadata { diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 405380f2..378ca3b1 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -13,6 +13,7 @@ case object HTIFNCores extends Field[Int] abstract trait HTIFParameters extends UsesParameters { val dataBits = params(TLDataBits) + val dataBeats = params(TLDataBeats) val co = params(TLCoherence) val w = params(HTIFWidth) val nSCR = params(HTIFNSCR) @@ -71,7 +72,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { // system is 'interesting' if any tile is 'interesting' val short_request_bits = 64 - val long_request_bits = short_request_bits + dataBits + val long_request_bits = short_request_bits + dataBits*dataBeats require(short_request_bits % w == 0) val rx_count_w = 13 + log2Up(64) - log2Up(w) // data size field is 12 bits @@ -150,12 +151,13 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { state_tx))) } - val acq_q = Module(new Queue(new Acquire, 1)) - when (state === state_mem_wreq && acq_q.io.enq.ready) { - state := state_mem_wresp + val (cnt, cnt_done) = Counter((state === state_mem_wreq && io.mem.acquire.ready) || + (state === state_mem_rresp && io.mem.grant.valid), dataBeats) + when (state === state_mem_wreq) { + when (cnt_done) { state := state_mem_wresp } } - when (state === state_mem_rreq && acq_q.io.enq.ready) { - state := state_mem_rresp + when (state === state_mem_rreq) { + when(io.mem.acquire.ready) { state := state_mem_rresp } } when (state === state_mem_wresp) { when (mem_acked) { @@ -164,10 +166,10 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { } } when (state === state_mem_rresp) { - when (io.mem.grant.valid) { + when (cnt_done) { state := state_mem_finish + mem_acked := Bool(false) } - mem_acked := Bool(false) } when (state === state_mem_finish && io.mem.finish.ready) { state := Mux(cmd === cmd_readmem || pos === UInt(1), state_tx, state_rx) @@ -182,22 +184,19 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { state := Mux(cmd === cmd_readmem && pos != UInt(0), state_mem_rreq, state_rx) } - var mem_req_data: Bits = null + var mem_req_data: UInt = null for (i <- 0 until dataBits/short_request_bits) { - val idx = UInt(i, log2Up(dataBits/short_request_bits)) + val idx = Cat(cnt, UInt(i, log2Up(dataBits/short_request_bits))) when (state === state_mem_rresp && io.mem.grant.valid) { packet_ram(idx) := io.mem.grant.bits.payload.data((i+1)*short_request_bits-1, i*short_request_bits) } mem_req_data = Cat(packet_ram(idx), mem_req_data) } - acq_q.io.enq.valid := state === state_mem_rreq || state === state_mem_wreq val init_addr = addr.toUInt >> UInt(offsetBits-3) - acq_q.io.enq.bits := Mux(cmd === cmd_writemem, - UncachedWrite(init_addr, UInt(0)), + io.mem.acquire.valid := state === state_mem_rreq || state === state_mem_wreq + io.mem.acquire.bits.payload := Mux(cmd === cmd_writemem, + UncachedWrite(init_addr, mem_req_data), UncachedRead(init_addr)) - io.mem.acquire.valid := acq_q.io.deq.valid - acq_q.io.deq.ready := io.mem.acquire.ready - io.mem.acquire.bits.payload := acq_q.io.deq.bits io.mem.acquire.bits.payload.data := mem_req_data io.mem.acquire.bits.header.src := UInt(params(LNClients)) // By convention HTIF is the client with the largest id io.mem.acquire.bits.header.dst := UInt(0) // DNC; Overwritten outside module @@ -255,7 +254,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { for (i <- 0 until scr_rdata.size) scr_rdata(i) := io.scr.rdata(i) scr_rdata(0) := UInt(nCores) - scr_rdata(1) := UInt((BigInt(dataBits/8) << acq_q.io.enq.bits.addr.getWidth) >> 20) + scr_rdata(1) := UInt((BigInt(dataBits*dataBeats/8) << params(TLAddrBits)) >> 20) io.scr.wen := Bool(false) io.scr.wdata := pcr_wdata diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index ea41256e..d6133fc0 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -209,81 +209,122 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { val mem = new MemIO } val co = params(TLCoherence) - val tbits = params(MIFTagBits) - val dbits = params(MIFDataBits) - val dbeats = params(MIFDataBeats) - require(params(TLDataBits) == dbits*dbeats) + val mifTagBits = params(MIFTagBits) + val mifDataBits = params(MIFDataBits) + val mifDataBeats = params(MIFDataBeats) + val tlDataBits = params(TLDataBits) + val tlDataBeats = params(TLDataBeats) + val dataBits = tlDataBits*tlDataBeats + require(tlDataBits*tlDataBeats == mifDataBits*mifDataBeats) //require(params(TLClientXactIdBits) <= params(MIFTagBits)) + // Decompose outgoing TL Acquires into MemIO cmd and data val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth)) val mem_data_q = Module(new Queue(new MemData, qDepth)) - val cnt_max = dbeats - val cnt_out = Reg(UInt(width = log2Up(cnt_max+1))) + io.uncached.acquire.ready := Bool(false) + io.uncached.grant.valid := Bool(false) + io.mem.resp.ready := Bool(false) + mem_cmd_q.io.enq.valid := Bool(false) + mem_data_q.io.enq.valid := Bool(false) + + val acq_has_data = co.messageHasData(io.uncached.acquire.bits.payload) + val (tl_cnt_out, tl_wrap_out) = Counter(io.uncached.acquire.fire() && acq_has_data, tlDataBeats) + val (mif_cnt_out, mif_wrap_out) = Counter(mem_data_q.io.enq.fire(), mifDataBeats) val active_out = Reg(init=Bool(false)) val cmd_sent_out = Reg(init=Bool(false)) - val buf_out = Reg(Bits()) + val tl_done_out = Reg(init=Bool(false)) + val mif_done_out = Reg(init=Bool(false)) val tag_out = Reg(Bits()) val addr_out = Reg(Bits()) val has_data = Reg(init=Bool(false)) + val tl_buf_out = Vec.fill(tlDataBeats){ Reg(io.uncached.acquire.bits.payload.data.clone) } + val mif_buf_out = Vec.fill(mifDataBeats){ new MemData } + mif_buf_out := mif_buf_out.fromBits(tl_buf_out.toBits) + val mif_prog_out = (mif_cnt_out+UInt(1, width = log2Up(mifDataBeats+1)))*UInt(mifDataBits) + val tl_prog_out = tl_cnt_out*UInt(tlDataBits) - val cnt_in = Reg(UInt(width = log2Up(cnt_max+1))) - val active_in = Reg(init=Bool(false)) - val buf_in = Reg(Bits()) - val tag_in = Reg(UInt(width = tbits)) - - // Decompose outgoing TL Acquires into MemIO cmd and data - when(!active_out && io.uncached.acquire.valid) { - active_out := Bool(true) - cmd_sent_out := Bool(false) - cnt_out := UInt(0) - buf_out := io.uncached.acquire.bits.payload.data - tag_out := io.uncached.acquire.bits.payload.client_xact_id - addr_out := io.uncached.acquire.bits.payload.addr - has_data := co.messageHasData(io.uncached.acquire.bits.payload) + when(!active_out){ + io.uncached.acquire.ready := Bool(true) + when(io.uncached.acquire.valid) { + active_out := Bool(true) + cmd_sent_out := Bool(false) + tag_out := io.uncached.acquire.bits.payload.client_xact_id + addr_out := io.uncached.acquire.bits.payload.addr + has_data := acq_has_data + tl_done_out := tl_wrap_out + mif_done_out := Bool(false) + tl_buf_out(tl_cnt_out) := io.uncached.acquire.bits.payload.data + } } when(active_out) { + when(!cmd_sent_out) { + mem_cmd_q.io.enq.valid := Bool(true) + } + when(has_data) { + when(!tl_done_out) { + io.uncached.acquire.ready := Bool(true) + when(io.uncached.acquire.valid) { + tl_buf_out(tl_cnt_out) := io.uncached.acquire.bits.payload.data + } + } + when(!mif_done_out) { + mem_data_q.io.enq.valid := tl_done_out || mif_prog_out <= tl_prog_out + } + } when(mem_cmd_q.io.enq.fire()) { cmd_sent_out := Bool(true) } - when(mem_data_q.io.enq.fire()) { - cnt_out := cnt_out + UInt(1) - buf_out := buf_out >> UInt(dbits) - } - when(cmd_sent_out && (!has_data || cnt_out === UInt(cnt_max))) { + when(tl_wrap_out) { tl_done_out := Bool(true) } + when(mif_wrap_out) { mif_done_out := Bool(true) } + when(cmd_sent_out && (!has_data || mif_done_out)) { active_out := Bool(false) } } - io.uncached.acquire.ready := !active_out - mem_cmd_q.io.enq.valid := active_out && !cmd_sent_out mem_cmd_q.io.enq.bits.rw := has_data mem_cmd_q.io.enq.bits.tag := tag_out mem_cmd_q.io.enq.bits.addr := addr_out - mem_data_q.io.enq.valid := active_out && has_data && cnt_out < UInt(cnt_max) - mem_data_q.io.enq.bits.data := buf_out + mem_data_q.io.enq.bits.data := mif_buf_out(mif_cnt_out).data io.mem.req_cmd <> mem_cmd_q.io.deq io.mem.req_data <> mem_data_q.io.deq // Aggregate incoming MemIO responses into TL Grants - io.mem.resp.ready := !active_in || cnt_in < UInt(cnt_max) - io.uncached.grant.valid := active_in && (cnt_in === UInt(cnt_max)) - io.uncached.grant.bits.payload := Grant(Bool(true), Grant.uncachedRead, tag_in, UInt(0), buf_in) - when(!active_in && io.mem.resp.valid) { - active_in := Bool(true) - cnt_in := UInt(1) - buf_in := io.mem.resp.bits.data << UInt(dbits*(cnt_max-1)) - tag_in := io.mem.resp.bits.tag + val (mif_cnt_in, mif_wrap_in) = Counter(io.mem.resp.fire(), mifDataBeats) // TODO: Assumes all resps have data + val (tl_cnt_in, tl_wrap_in) = Counter(io.uncached.grant.fire(), tlDataBeats) + val active_in = Reg(init=Bool(false)) + val mif_done_in = Reg(init=Bool(false)) + val tag_in = Reg(UInt(width = mifTagBits)) + val mif_buf_in = Vec.fill(mifDataBeats){ Reg(new MemData) } + val tl_buf_in = Vec.fill(tlDataBeats){ io.uncached.acquire.bits.payload.data.clone } + tl_buf_in := tl_buf_in.fromBits(mif_buf_in.toBits) + val tl_prog_in = (tl_cnt_in+UInt(1, width = log2Up(tlDataBeats+1)))*UInt(tlDataBits) + val mif_prog_in = mif_cnt_in*UInt(mifDataBits) + + when(!active_in) { + io.mem.resp.ready := Bool(true) + when(io.mem.resp.valid) { + active_in := Bool(true) + mif_done_in := mif_wrap_in + tag_in := io.mem.resp.bits.tag + mif_buf_in(tl_cnt_in).data := io.mem.resp.bits.data + } } + when(active_in) { - when(io.uncached.grant.fire()) { - active_in := Bool(false) - } - when(io.mem.resp.fire()) { - buf_in := Cat(io.mem.resp.bits.data, buf_in(cnt_max*dbits-1,dbits)) - cnt_in := cnt_in + UInt(1) + io.uncached.grant.valid := mif_done_in || tl_prog_in <= mif_prog_in + when(!mif_done_in) { + io.mem.resp.ready := Bool(true) + when(io.mem.resp.valid) { + mif_buf_in(mif_cnt_in).data := io.mem.resp.bits.data + } } + when(mif_wrap_in) { mif_done_in := Bool(true) } + when(tl_wrap_in) { active_in := Bool(false) } } + + io.uncached.grant.bits.payload := Grant(Bool(true), Grant.uncachedRead, tag_in, UInt(0), + tl_buf_in(tl_cnt_in)) } class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module @@ -390,8 +431,8 @@ class MemPipeIOUncachedTileLinkIOConverter(outstanding: Int, refillCycles: Int) val a = Module(new MemIOUncachedTileLinkIOConverter(2)) val b = Module(new MemPipeIOMemIOConverter(outstanding, refillCycles)) a.io.uncached <> io.uncached - b.io.cpu.req_cmd <> Queue(a.io.mem.req_cmd, 2) - b.io.cpu.req_data <> Queue(a.io.mem.req_data, refillCycles) + b.io.cpu.req_cmd <> Queue(a.io.mem.req_cmd, 2, pipe=true) + b.io.cpu.req_data <> Queue(a.io.mem.req_data, refillCycles, pipe=true) a.io.mem.resp <> b.io.cpu.resp b.io.mem <> io.mem } diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 1e535507..678efb6a 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -26,9 +26,11 @@ abstract trait TileLinkParameters extends UsesParameters { (tlSubblockAddrBits + tlUncachedOperandSizeBits + tlAtomicOpcodeBits)) + val co = params(TLCoherence) } -class TLBundle extends Bundle with TileLinkParameters +abstract class TLBundle extends Bundle with TileLinkParameters +abstract class TLModule extends Module with TileLinkParameters trait HasPhysicalAddress extends TLBundle { val addr = UInt(width = tlAddrBits) @@ -55,7 +57,7 @@ class Acquire extends ClientSourcedMessage with HasClientTransactionId with HasTileLinkData { val uncached = Bool() - val a_type = UInt(width = max(log2Up(Acquire.nUncachedAcquireTypes), params(TLCoherence).acquireTypeWidth)) + val a_type = UInt(width = max(log2Up(Acquire.nUncachedAcquireTypes), co.acquireTypeWidth)) val subblock = Bits(width = tlSubblockUnionBits) val sbAddrOff = tlSubblockAddrBits + tlUncachedOperandSizeBits val opSzOff = tlUncachedOperandSizeBits + sbAddrOff @@ -147,7 +149,7 @@ object Probe { class Probe extends MasterSourcedMessage with HasPhysicalAddress { - val p_type = UInt(width = params(TLCoherence).probeTypeWidth) + val p_type = UInt(width = co.probeTypeWidth) def is(t: UInt) = p_type === t } @@ -172,7 +174,7 @@ class Release extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasTileLinkData { - val r_type = UInt(width = params(TLCoherence).releaseTypeWidth) + val r_type = UInt(width = co.releaseTypeWidth) def is(t: UInt) = r_type === t } @@ -181,7 +183,7 @@ class Grant extends MasterSourcedMessage with HasClientTransactionId with HasMasterTransactionId { val uncached = Bool() - val g_type = UInt(width = max(log2Up(Grant.nUncachedGrantTypes), params(TLCoherence).grantTypeWidth)) + val g_type = UInt(width = max(log2Up(Grant.nUncachedGrantTypes), co.grantTypeWidth)) def is(t: UInt) = g_type === t } @@ -221,7 +223,7 @@ class TileLinkIO extends UncachedTileLinkIO { val release = new DecoupledIO(new LogicalNetworkIO(new Release)) } -abstract class TileLinkArbiterLike(val arbN: Int) extends Module { +abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule { type MasterSourcedWithId = MasterSourcedMessage with HasClientTransactionId type ClientSourcedWithId = ClientSourcedMessage with HasClientTransactionId @@ -232,8 +234,8 @@ abstract class TileLinkArbiterLike(val arbN: Int) extends Module { def hookupClientSource[M <: ClientSourcedWithId] (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], out: DecoupledIO[LogicalNetworkIO[M]]) { - def hasData(m: LogicalNetworkIO[M]) = params(TLCoherence).messageHasData(m.payload) - val arb = Module(new RRArbiter(out.bits.clone, arbN)) + def hasData(m: LogicalNetworkIO[M]) = co.messageHasData(m.payload) + val arb = Module(new LockingRRArbiter(out.bits.clone, arbN, params(TLDataBeats), Some(hasData _))) out <> arb.io.out ins.zipWithIndex.zip(arb.io.in).map{ case ((req,id), arb) => { arb.valid := req.valid diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 858d603c..69b13800 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -8,14 +8,16 @@ case object NAcquireTransactors extends Field[Int] case object L2StoreDataQueueDepth extends Field[Int] case object NClients extends Field[Int] -abstract trait CoherenceAgentParameters extends UsesParameters { - val co = params(TLCoherence) +abstract trait CoherenceAgentParameters extends UsesParameters + with TileLinkParameters { val nReleaseTransactors = 1 val nAcquireTransactors = params(NAcquireTransactors) val nTransactors = nReleaseTransactors + nAcquireTransactors val nClients = params(NClients) - val sdqDepth = params(L2StoreDataQueueDepth) - val sdqIdBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(params(L2StoreDataQueueDepth))) + 1 + val sdqDepth = params(L2StoreDataQueueDepth)*tlDataBeats + val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(params(L2StoreDataQueueDepth))) + + log2Ceil(tlDataBeats) + val nDataQueueLocations = 3 //Stores, VoluntaryWBs, Releases } abstract class CoherenceAgent(innerId: String, outerId: String) extends Module @@ -27,75 +29,93 @@ abstract class CoherenceAgent(innerId: String, outerId: String) extends Module } } +class DataQueueLocation extends Bundle with CoherenceAgentParameters { + val idx = UInt(width = dqIdxBits) + val loc = UInt(width = log2Ceil(nDataQueueLocations)) +} +object DataQueueLocation { + def apply(idx: UInt, loc: UInt) = { + val d = new DataQueueLocation + d.idx := idx + d.loc := loc + d + } +} + class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends CoherenceAgent(innerId, outerId) { - // Queue to store impending UncachedWrite data - val sdq_val = Reg(init=Bits(0, sdqDepth)) - val sdq_alloc_id = PriorityEncoder(~sdq_val(sdqDepth-1,0)) - val sdq_rdy = !sdq_val.andR - val sdq_enq = io.inner.acquire.valid && io.inner.acquire.ready && co.messageHasData(io.inner.acquire.bits.payload) - val sdq = Vec.fill(sdqDepth){Reg(io.inner.acquire.bits.payload.data)} - when (sdq_enq) { sdq(sdq_alloc_id) := io.inner.acquire.bits.payload.data } + val internalDataBits = new DataQueueLocation().getWidth + val inStoreQueue :: inVolWBQueue :: inClientReleaseQueue :: Nil = Enum(UInt(), nDataQueueLocations) // Create SHRs for outstanding transactions val trackerList = (0 until nReleaseTransactors).map(id => - Module(new VoluntaryReleaseTracker(id, bankId, innerId, outerId), {case TLDataBits => sdqIdBits})) ++ + Module(new VoluntaryReleaseTracker(id, bankId, innerId, outerId), {case TLDataBits => internalDataBits})) ++ (nReleaseTransactors until nTransactors).map(id => - Module(new AcquireTracker(id, bankId, innerId, outerId), {case TLDataBits => sdqIdBits})) + Module(new AcquireTracker(id, bankId, innerId, outerId), {case TLDataBits => internalDataBits})) // Propagate incoherence flags trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) - // Handle acquire transaction initiation + // Queue to store impending UncachedWrite data val acquire = io.inner.acquire + val sdq_val = Reg(init=Bits(0, sdqDepth)) + val sdq_alloc_id = PriorityEncoder(~sdq_val) + val sdq_rdy = !sdq_val.andR + val sdq_enq = acquire.fire() && co.messageHasData(acquire.bits.payload) + val sdq = Vec.fill(sdqDepth){ Reg(io.inner.acquire.bits.payload.data) } + when (sdq_enq) { sdq(sdq_alloc_id) := acquire.bits.payload.data } + + // Handle acquire transaction initiation val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val block_acquires = any_acquire_conflict - val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.inner alloc_arb.io.in(i).valid := t.acquire.ready t.acquire.bits := acquire.bits - t.acquire.bits.payload.data := Cat(sdq_alloc_id, UInt(1)) + t.acquire.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits t.acquire.valid := alloc_arb.io.in(i).ready } acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && sdq_rdy && !block_acquires alloc_arb.io.out.ready := acquire.valid && sdq_rdy && !block_acquires - // Handle probe request generation - val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) - io.inner.probe <> probe_arb.io.out - probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.probe } - - // Handle releases, which might be voluntary and might have data + // Queue to store impending Voluntary Release data val release = io.inner.release val voluntary = co.isVoluntary(release.bits.payload) + val vwbdq_enq = release.fire() && voluntary && co.messageHasData(release.bits.payload) + val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, tlDataBeats) //TODO Zero width + val vwbdq = Vec.fill(tlDataBeats){ Reg(release.bits.payload.data) } //TODO Assumes nReleaseTransactors == 1 + when(vwbdq_enq) { vwbdq(rel_data_cnt) := release.bits.payload.data } + + // Handle releases, which might be voluntary and might have data val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) val block_releases = Bool(false) val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b} val release_idx = Mux(voluntary, UInt(0), conflict_idx) - // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.inner t.release.bits := release.bits - t.release.bits.payload.data := (if (i < nReleaseTransactors) Cat(UInt(i), UInt(2)) else UInt(0)) + t.release.bits.payload.data := (if (i < nReleaseTransactors) + DataQueueLocation(rel_data_cnt, inVolWBQueue) + else DataQueueLocation(UInt(0), inClientReleaseQueue)).toBits t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases } release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases - val vwbdq = Vec.fill(nReleaseTransactors){ Reg(release.bits.payload.data) } - when(voluntary && release.fire()) { - vwbdq(release_idx) := release.bits.payload.data - } + // Wire probe requests to clients + val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) + io.inner.probe <> probe_arb.io.out + probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.probe } - // Reply to initial requestor - val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) + // Wire grant reply to initiating client + def hasData(m: LogicalNetworkIO[Grant]) = co.messageHasData(m.payload) + val grant_arb = Module(new LockingArbiter(new LogicalNetworkIO(new Grant), trackerList.size, tlDataBeats, Some(hasData _))) io.inner.grant.bits.payload.data := io.outer.grant.bits.payload.data io.inner.grant <> grant_arb.io.out grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } - // Free finished transactions + // Wire finished transaction acks val ack = io.inner.finish trackerList.map(_.io.inner.finish.valid := ack.valid) trackerList.map(_.io.inner.finish.bits := ack.bits) @@ -103,27 +123,28 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends // Create an arbiter for the one memory port val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), - { case TLId => outerId; case TLDataBits => sdqIdBits }) + { case TLId => outerId; case TLDataBits => internalDataBits }) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } - val is_in_sdq = outer_arb.io.out.acquire.bits.payload.data(0) - val is_in_vwbdq = outer_arb.io.out.acquire.bits.payload.data(1) - val free_sdq_id = outer_arb.io.out.acquire.bits.payload.data >> UInt(1) - val free_vwbdq_id = outer_arb.io.out.acquire.bits.payload.data >> UInt(2) - val free_sdq = io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload) && is_in_sdq - io.outer.acquire.bits.payload.data := Mux(is_in_sdq, sdq(free_sdq_id), - Mux(is_in_vwbdq, vwbdq(free_vwbdq_id), release.bits.payload.data)) + val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.payload.data) + val is_in_sdq = outer_data_ptr.loc === inStoreQueue + val free_sdq = io.outer.acquire.fire() && + co.messageHasData(io.outer.acquire.bits.payload) && + outer_data_ptr.loc === inStoreQueue + io.outer.acquire.bits.payload.data := MuxLookup(outer_data_ptr.loc, release.bits.payload.data, Array( + inStoreQueue -> sdq(outer_data_ptr.idx), + inVolWBQueue -> vwbdq(outer_data_ptr.idx))) io.outer <> outer_arb.io.out // Update SDQ valid bits when (io.outer.acquire.valid || sdq_enq) { - sdq_val := sdq_val & ~(UIntToOH(free_sdq_id) & Fill(sdqDepth, free_sdq)) | + sdq_val := sdq_val & ~(UIntToOH(outer_data_ptr.idx) & Fill(sdqDepth, free_sdq)) | PriorityEncoderOH(~sdq_val(sdqDepth-1,0)) & Fill(sdqDepth, sdq_enq) } } abstract class XactTracker(innerId: String, outerId: String) extends Module { - val (co, nClients) = (params(TLCoherence),params(NClients)) + val (co, nClients, tlDataBeats) = (params(TLCoherence),params(NClients),params(TLDataBeats)) val io = new Bundle { val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) @@ -137,13 +158,20 @@ abstract class XactTracker(innerId: String, outerId: String) extends Module { val c_gnt = io.inner.grant.bits val c_ack = io.inner.finish.bits val m_gnt = io.outer.grant.bits + } class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { - val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) + val s_idle :: s_outer :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) val xact = Reg{ new Release } val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) + val data_ptrs = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) } + val collect_inner_data = Reg(init=Bool(false)) + val (inner_data_cnt, inner_data_done) = + Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) + val (outer_data_cnt, outer_data_done) = + Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) io.has_acquire_conflict := Bool(false) io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && @@ -156,7 +184,7 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute io.outer.acquire.bits.payload := Bundle(UncachedWrite( xact.addr, UInt(trackerId), - xact.data), + data_ptrs(outer_data_cnt)), { case TLId => outerId }) io.inner.acquire.ready := Bool(false) io.inner.probe.valid := Bool(false) @@ -169,18 +197,28 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute xact.client_xact_id, UInt(trackerId)) + when(collect_inner_data) { + io.inner.release.ready := Bool(true) + when(io.inner.release.valid) { + data_ptrs(inner_data_cnt) := c_rel.payload.data + } + when(inner_data_done) { collect_inner_data := Bool(false) } + } + switch (state) { is(s_idle) { + io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { - io.inner.release.ready := Bool(true) xact := c_rel.payload init_client_id := c_rel.header.src - state := Mux(co.messageHasData(c_rel.payload), s_mem, s_ack) + data_ptrs(UInt(0)) := c_rel.payload.data + collect_inner_data := co.messageHasData(c_rel.payload) + state := Mux(co.messageHasData(c_rel.payload), s_outer, s_ack) } } - is(s_mem) { - io.outer.acquire.valid := Bool(true) - when(io.outer.acquire.ready) { state := s_ack } + is(s_outer) { + io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) + when(outer_data_done) { state := s_ack } } is(s_ack) { io.inner.grant.valid := Bool(true) @@ -195,6 +233,12 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val xact = Reg{ new Acquire } val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) //TODO: Will need id reg for merged release xacts + val data_ptrs = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } + val collect_inner_data = Reg(init=Bool(false)) + val (inner_data_cnt, inner_data_done) = + Counter(io.inner.acquire.fire() && co.messageHasData(io.inner.acquire.bits.payload), tlDataBeats) + val (outer_data_cnt, outer_data_done) = + Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) val release_count = if (nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(nClients))) val probe_flags = Reg(init=Bits(0, width = nClients)) @@ -202,23 +246,21 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val pending_outer_write = co.messageHasData(xact) val pending_outer_read = co.requiresOuterRead(xact, co.masterMetadataOnFlush) - val outer_write_acq = Bundle(UncachedWrite(xact.addr, UInt(trackerId), xact.data), + val outer_write_acq = Bundle(UncachedWrite(xact.addr, UInt(trackerId), data_ptrs(outer_data_cnt)), { case TLId => outerId }) - val outer_write_rel = Bundle(UncachedWrite(xact.addr, UInt(trackerId), UInt(0)), // Special SQDId + val outer_write_rel = Bundle(UncachedWrite(xact.addr, UInt(trackerId), c_rel.payload.data), { case TLId => outerId }) val outer_read = Bundle(UncachedRead(xact.addr, UInt(trackerId)), { case TLId => outerId }) val probe_initial_flags = Bits(width = nClients) probe_initial_flags := Bits(0) - if (nClients > 1) { - // issue self-probes for uncached read xacts to facilitate I$ coherence - val probe_self = Bool(true) //co.needsSelfProbe(io.inner.acquire.bits.payload) - val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(nClients)-1,0))) - probe_initial_flags := ~(io.tile_incoherent | myflag) - } + // issue self-probes for uncached read xacts to facilitate I$ coherence + val probe_self = co.requiresSelfProbe(io.inner.acquire.bits.payload) + val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(nClients)-1,0))) + probe_initial_flags := ~(io.tile_incoherent | myflag) - io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) + io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) && !collect_inner_data io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) io.outer.acquire.valid := Bool(false) @@ -244,6 +286,14 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) + when(collect_inner_data) { + io.inner.acquire.ready := Bool(true) + when(io.inner.acquire.valid) { + data_ptrs(inner_data_cnt) := c_acq.payload.data + } + when(inner_data_done) { collect_inner_data := Bool(false) } + } + switch (state) { is(s_idle) { io.inner.acquire.ready := Bool(true) @@ -252,14 +302,13 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src + data_ptrs(UInt(0)) := c_acq.payload.data + collect_inner_data := co.messageHasData(c_acq.payload) probe_flags := probe_initial_flags - if(nClients > 1) { - release_count := PopCount(probe_initial_flags) - state := Mux(probe_initial_flags.orR, s_probe, - Mux(needs_outer_write, s_mem_write, - Mux(needs_outer_read, s_mem_read, s_make_grant))) - } else state := Mux(needs_outer_write, s_mem_write, - Mux(needs_outer_read, s_mem_read, s_make_grant)) + release_count := PopCount(probe_initial_flags) + state := Mux(probe_initial_flags.orR, s_probe, + Mux(needs_outer_write, s_mem_write, + Mux(needs_outer_read, s_mem_read, s_make_grant))) } } is(s_probe) { @@ -276,15 +325,17 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri io.outer.acquire.bits.payload := outer_write_rel when(io.outer.acquire.ready) { io.inner.release.ready := Bool(true) - if(nClients > 1) release_count := release_count - UInt(1) - when(release_count === UInt(1)) { - state := Mux(pending_outer_write, s_mem_write, - Mux(pending_outer_read, s_mem_read, s_make_grant)) + when(outer_data_done) { + release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } } } } .otherwise { io.inner.release.ready := Bool(true) - if(nClients > 1) release_count := release_count - UInt(1) + release_count := release_count - UInt(1) when(release_count === UInt(1)) { state := Mux(pending_outer_write, s_mem_write, Mux(pending_outer_read, s_mem_read, s_make_grant)) @@ -300,9 +351,9 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri } } is(s_mem_write) { - io.outer.acquire.valid := Bool(true) + io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) io.outer.acquire.bits.payload := outer_write_acq - when(io.outer.acquire.ready) { + when(outer_data_done) { state := Mux(pending_outer_read, s_mem_read, s_make_grant) } } diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util.scala new file mode 100644 index 00000000..c28778aa --- /dev/null +++ b/uncore/src/main/scala/util.scala @@ -0,0 +1,88 @@ +// See LICENSE for license details. + +package uncore + +import Chisel._ +import scala.math._ + +object MuxBundle { + def apply[T <: Data] (default: T, mapping: Seq[(Bool, T)]): T = { + mapping.reverse.foldLeft(default)((b, a) => Mux(a._1, a._2, b)) + } +} + +// Produces 0-width value when counting to 1 +class ZCounter(val n: Int) { + val value = Reg(init=UInt(0, log2Ceil(n))) + def inc(): Bool = { + if (n == 1) Bool(true) + else { + val wrap = value === UInt(n-1) + value := Mux(Bool(!isPow2(n)) && wrap, UInt(0), value + UInt(1)) + wrap + } + } +} + +object ZCounter { + def apply(n: Int) = new ZCounter(n) + def apply(cond: Bool, n: Int): (UInt, Bool) = { + val c = new ZCounter(n) + var wrap: Bool = null + when (cond) { wrap = c.inc() } + (c.value, cond && wrap) + } +} + +class FlowThroughSerializer[T <: HasTileLinkData](gen: LogicalNetworkIO[T], n: Int, doSer: T => Bool) extends Module { + val io = new Bundle { + val in = Decoupled(gen.clone).flip + val out = Decoupled(gen.clone) + val cnt = UInt(OUTPUT, log2Up(n)) + val done = Bool(OUTPUT) + } + val narrowWidth = io.in.bits.payload.data.getWidth / n + require(io.in.bits.payload.data.getWidth % narrowWidth == 0) + + if(n == 1) { + io.in <> io.out + io.cnt := UInt(width = 0) + io.done := Bool(true) + } else { + val cnt = Reg(init=UInt(0, width = log2Up(n))) + val wrap = cnt === UInt(n-1) + val rbits = Reg(init=io.in.bits) + val active = Reg(init=Bool(false)) + + val shifter = Vec.fill(n){Bits(width = narrowWidth)} + (0 until n).foreach { + i => shifter(i) := rbits.payload.data((i+1)*narrowWidth-1,i*narrowWidth) + } + + io.done := Bool(false) + io.cnt := cnt + io.in.ready := !active + io.out.valid := active || io.in.valid + io.out.bits := io.in.bits + when(!active && io.in.valid) { + when(doSer(io.in.bits.payload)) { + cnt := Mux(io.out.ready, UInt(1), UInt(0)) + rbits := io.in.bits + active := Bool(true) + } + io.done := !doSer(io.in.bits.payload) + } + when(active) { + io.out.bits := rbits + io.out.bits.payload.data := shifter(cnt) + when(io.out.ready) { + cnt := cnt + UInt(1) + when(wrap) { + cnt := UInt(0) + io.done := Bool(true) + active := Bool(false) + } + } + } + } +} From 424df2368f6c93f34800f61bf7789d84d7762663 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 12 Dec 2014 01:11:08 -0800 Subject: [PATCH 266/374] 1R/W L2 data array? Add TLDataBeats to new LLC; all bmarks pass --- uncore/src/main/scala/cache.scala | 335 +++++++++++++++++--------- uncore/src/main/scala/coherence.scala | 26 +- uncore/src/main/scala/tilelink.scala | 9 +- uncore/src/main/scala/uncore.scala | 142 ++++++----- 4 files changed, 308 insertions(+), 204 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index b96aab70..97a0c9a5 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -134,6 +134,7 @@ class L2MetaWriteReq extends MetaWriteReq[L2Metadata](new L2Metadata) with HasL2Id { override def clone = new L2MetaWriteReq().asInstanceOf[this.type] } + class L2MetaResp extends L2HellaCacheBundle with HasL2Id with HasL2InternalRequestState @@ -156,10 +157,10 @@ class L2MetadataArray extends L2HellaCacheModule { meta.io.read <> io.read meta.io.write <> io.write - val s1_clk_en = Reg(next = io.read.fire()) val s1_tag = RegEnable(io.read.bits.tag, io.read.valid) val s1_id = RegEnable(io.read.bits.id, io.read.valid) def wayMap[T <: Data](f: Int => T) = Vec((0 until nWays).map(f)) + val s1_clk_en = Reg(next = io.read.fire()) val s1_tag_eq_way = wayMap((w: Int) => meta.io.resp(w).tag === s1_tag) val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && co.isValid(meta.io.resp(w).coh)).toBits val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) @@ -220,11 +221,11 @@ class L2DataArray extends L2HellaCacheModule { } array(RegEnable(raddr, io.read.bits.way_en(w) && io.read.valid)) } - io.resp.valid := ShiftRegister(io.read.valid, 2) - io.resp.bits.id := ShiftRegister(io.read.bits.id, 2) - io.resp.bits.data := Mux1H(ShiftRegister(io.read.bits.way_en, 2), resp) + io.resp.valid := ShiftRegister(io.read.fire(), 1) + io.resp.bits.id := ShiftRegister(io.read.bits.id, 1) + io.resp.bits.data := Mux1H(ShiftRegister(io.read.bits.way_en, 1), resp) - io.read.ready := Bool(true) + io.read.ready := !io.write.valid // TODO 1R/W vs 1R1W? io.write.ready := Bool(true) } @@ -233,7 +234,6 @@ class L2HellaCache(bankId: Int, innerId: String, outerId: String) extends require(isPow2(nSets)) require(isPow2(nWays)) - require(refillCycles == 1) val tshrfile = Module(new TSHRFile(bankId, innerId, outerId)) val meta = Module(new L2MetadataArray) @@ -257,8 +257,11 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac } // Wiring helper funcs - def doOutputArbitration[T <: Data](out: DecoupledIO[T], ins: Seq[DecoupledIO[T]]) { - val arb = Module(new RRArbiter(out.bits.clone, ins.size)) + def doOutputArbitration[T <: Data](out: DecoupledIO[T], + ins: Seq[DecoupledIO[T]], + count: Int = 1, + lock: T => Bool = (a: T) => Bool(true)) { + val arb = Module(new LockingRRArbiter(out.bits.clone, ins.size, count, lock)) out <> arb.io.out arb.io.in zip ins map { case (a, in) => a <> in } } @@ -282,7 +285,6 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac val acquire = io.inner.acquire val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val block_acquires = any_acquire_conflict - val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.inner @@ -293,9 +295,6 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && !block_acquires alloc_arb.io.out.ready := acquire.valid && !block_acquires - // Handle probe requests - doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) - // Handle releases, which might be voluntary and might have data val release = io.inner.release val voluntary = co.isVoluntary(release.bits.payload) @@ -310,10 +309,7 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac } release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases - // Reply to initial requestor - doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) - - // Free finished transactions on ack + // Wire finished transaction acks val finish = io.inner.finish val finish_idx = finish.bits.payload.master_xact_id trackerList.zipWithIndex.map { case (t, i) => @@ -322,20 +318,26 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac trackerList.map(_.io.inner.finish.bits := finish.bits) finish.ready := Vec(trackerList.map(_.io.inner.finish.ready)).read(finish_idx) - // Arbitrate for the outer memory port + // Wire probe requests to clients + doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) + + // Wire grant reply to initiating client + def hasData(m: LogicalNetworkIO[Grant]) = co.messageHasData(m.payload) + doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant), tlDataBeats, hasData _) + + // Create an arbiter for the one memory port val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), {case TLId => outerId}) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } io.outer <> outer_arb.io.out - // Local memory + // Wire local memories doOutputArbitration(io.meta.read, trackerList.map(_.io.meta.read)) doOutputArbitration(io.meta.write, trackerList.map(_.io.meta.write)) doOutputArbitration(io.data.read, trackerList.map(_.io.data.read)) doOutputArbitration(io.data.write, trackerList.map(_.io.data.write)) doInputRouting(io.meta.resp, trackerList.map(_.io.meta.resp)) doInputRouting(io.data.resp, trackerList.map(_.io.data.resp)) - } @@ -356,21 +358,34 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa val c_ack = io.inner.finish.bits val m_gnt = io.outer.grant.bits - def mergeData(acq: Acquire, data: UInt): UInt = { - //TODO wite mask - Mux(co.messageHasData(acq), acq.data, data) + def mergeData(acq: Acquire, old_data: UInt, new_data: UInt): UInt = { + //TODO apply acq's write mask + Mux(co.messageHasData(acq), old_data, new_data) } } class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 7) val state = Reg(init=s_idle) - val xact = Reg{ new Release } - val xact_internal = Reg{ new L2MetaResp } - val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) + + val xact_src = Reg(io.inner.release.bits.header.src.clone) + val xact_r_type = Reg(io.inner.release.bits.payload.r_type) + val xact_addr = Reg(io.inner.release.bits.payload.addr.clone) + val xact_client_xact_id = Reg(io.inner.release.bits.payload.client_xact_id.clone) + val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) } + val xact_tag_match = Reg{ Bool() } + val xact_meta = Reg{ new L2Metadata } + val xact_way_en = Reg{ Bits(width = nWays) } + val xact = Release(xact_r_type, xact_addr, xact_client_xact_id) + + val collect_inner_data = Reg(init=Bool(false)) + val (inner_data_cnt, inner_data_done) = + Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) + val (local_data_cnt, local_data_done) = + Counter(io.data.write.fire(), tlDataBeats) io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && + io.has_release_conflict := co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && (state != s_idle) io.outer.grant.ready := Bool(false) @@ -382,38 +397,50 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.inner.finish.ready := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := init_client_id + io.inner.grant.bits.header.dst := xact_src io.inner.grant.bits.payload := Grant(Bool(false), - co.getGrantTypeOnVoluntaryWriteback(xact_internal.meta.coh), - xact.client_xact_id, + co.getGrantTypeOnVoluntaryWriteback(xact_meta.coh), + xact_client_xact_id, UInt(trackerId)) io.data.read.valid := Bool(false) io.data.write.valid := Bool(false) io.data.write.bits.id := UInt(trackerId) - io.data.write.bits.way_en := xact_internal.way_en - io.data.write.bits.addr := xact.addr + io.data.write.bits.way_en := xact_way_en + io.data.write.bits.addr := Cat(xact_addr, local_data_cnt) io.data.write.bits.wmask := SInt(-1) - io.data.write.bits.data := xact.data + io.data.write.bits.data := xact_data(local_data_cnt) io.meta.read.valid := Bool(false) io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta.read.bits.tag := xact.addr >> UInt(untagBits) + io.meta.read.bits.idx := xact_addr(untagBits-1,blockOffBits) + io.meta.read.bits.tag := xact_addr >> UInt(untagBits) io.meta.write.valid := Bool(false) io.meta.write.bits.id := UInt(trackerId) - io.meta.write.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta.write.bits.way_en := xact_internal.way_en - io.meta.write.bits.data.tag := xact.addr >> UInt(untagBits) + io.meta.write.bits.idx := xact_addr(untagBits-1,blockOffBits) + io.meta.write.bits.way_en := xact_way_en + io.meta.write.bits.data.tag := xact_addr >> UInt(untagBits) io.meta.write.bits.data.coh := co.masterMetadataOnRelease(xact, - xact_internal.meta.coh, - init_client_id) + xact_meta.coh, + xact_src) + + when(collect_inner_data) { + io.inner.release.ready := Bool(true) + when(io.inner.release.valid) { + xact_data(inner_data_cnt) := c_rel.payload.data + } + when(inner_data_done) { collect_inner_data := Bool(false) } + } switch (state) { is(s_idle) { io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { - xact := c_rel.payload - init_client_id := c_rel.header.src + xact_src := c_rel.header.src + xact_r_type := c_rel.payload.r_type + xact_addr := c_rel.payload.addr + xact_client_xact_id := c_rel.payload.client_xact_id + xact_data(UInt(0)) := c_rel.payload.data + collect_inner_data := co.messageHasData(c_rel.payload) state := s_meta_read } } @@ -423,15 +450,18 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou } is(s_meta_resp) { when(io.meta.resp.valid) { - xact_internal := io.meta.resp.bits + xact_tag_match := io.meta.resp.bits.tag_match + xact_meta := io.meta.resp.bits.meta + xact_way_en := io.meta.resp.bits.way_en state := Mux(io.meta.resp.bits.tag_match, Mux(co.messageHasData(xact), s_data_write, s_meta_write), s_grant) } } is(s_data_write) { - io.data.write.valid := Bool(true) - when(io.data.write.ready) { state := s_meta_write } + io.data.write.valid := (if(tlDataBeats == 1) Bool(true) + else !collect_inner_data || (local_data_cnt < inner_data_cnt)) + when(local_data_done) { state := s_meta_write } } is(s_meta_write) { io.meta.write.valid := Bool(true) @@ -454,72 +484,100 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { val s_idle :: s_meta_read :: s_meta_resp :: s_probe :: s_data_read_wb :: s_data_resp_wb :: s_outer_write_wb :: s_outer_read :: s_outer_resp :: s_data_read_hit :: s_data_resp_hit :: s_data_write :: s_outer_write_acq :: s_meta_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 16) val state = Reg(init=s_idle) - val xact = Reg{ new Acquire } - val xact_internal = Reg{ new L2MetaResp } - val init_client_id = Reg(init = UInt(0, width = log2Up(nClients))) + + val xact_src = Reg(io.inner.acquire.bits.header.src.clone) + val xact_uncached = Reg(io.inner.acquire.bits.payload.uncached.clone) + val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone) + val xact_addr = Reg(io.inner.acquire.bits.payload.addr.clone) + val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone) + val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone) + val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } + val xact_tag_match = Reg{ Bool() } + val xact_meta = Reg{ new L2Metadata } + val xact_way_en = Reg{ Bits(width = nWays) } + val xact = Acquire(xact_uncached, xact_a_type, xact_addr, xact_client_xact_id, UInt(0), xact_subblock) + val crel_had_data = Reg(init = Bool(false)) val crel_was_voluntary = Reg(init = Bool(false)) val crel_wb_src = Reg(init = UInt(0, width = log2Up(nClients))) val crel_wb_g_type = Reg(init = UInt(0, width = co.grantTypeWidth)) - val wb_buffer = Reg{xact.data.clone} - val addr_wb = Cat(xact_internal.meta.tag, xact.addr(untagBits-1,blockOffBits)) + val wb_buffer = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } + val wb_addr = Cat(xact_meta.tag, xact_addr(untagBits-1,blockOffBits)) + + val collect_cacq_data = Reg(init=Bool(false)) + //TODO: zero width wires + val (cacq_data_cnt, cacq_data_done) = + Counter(io.inner.acquire.fire() && co.messageHasData(io.inner.acquire.bits.payload), tlDataBeats) + val (crel_data_cnt, crel_data_done) = + Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) + val (cgnt_data_cnt, cgnt_data_done) = + Counter(io.inner.grant.fire() && co.messageHasData(io.inner.grant.bits.payload), tlDataBeats) + val (outer_data_write_cnt, outer_data_write_done) = + Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) + val (outer_data_resp_cnt, outer_data_resp_done) = + Counter(io.outer.grant.fire() && co.messageHasData(io.outer.grant.bits.payload), tlDataBeats) + val (local_data_read_cnt, local_data_read_done) = Counter(io.data.read.fire(), tlDataBeats) + val (local_data_write_cnt, local_data_write_done) = Counter(io.data.write.fire(), tlDataBeats) + val (local_data_resp_cnt, local_data_resp_done) = Counter(io.data.resp.valid, tlDataBeats) val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) val pending_probes = Reg(init = co.dir().flush) val curr_p_id = co.dir().next(pending_probes) - val is_uncached = xact.uncached - val tag_match = xact_internal.tag_match - val needs_writeback = !tag_match && co.needsWriteback(xact_internal.meta.coh) - val is_hit = tag_match && co.isHit(xact, xact_internal.meta.coh) - val needs_probes = co.requiresProbes(xact, xact_internal.meta.coh) - //TODO: does allocate + val needs_writeback = !xact_tag_match && co.needsWriteback(xact_meta.coh) + val is_hit = xact_tag_match && co.isHit(xact, xact_meta.coh) + val needs_probes = co.requiresProbes(xact, xact_meta.coh) + //TODO: uncached does or does not allocate - io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && - (state != s_idle) //TODO: Also indexes + //TODO: Are there any races between lines with the same idx? + //TODO: Allow hit under miss for stores + io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && + xact.addr(untagBits-1,0) === c_acq.payload.addr(untagBits-1,0) && + (state != s_idle) && + !collect_cacq_data io.has_release_conflict := (co.isCoherenceConflict(xact.addr, c_rel.payload.addr) || - co.isCoherenceConflict(addr_wb, c_rel.payload.addr)) && - (state != s_idle) //TODO: Also indexes? + co.isCoherenceConflict(wb_addr, c_rel.payload.addr)) && + (state != s_idle) val next_coh_on_release = co.masterMetadataOnRelease( c_rel.payload, - xact_internal.meta.coh, + xact_meta.coh, c_rel.header.src) val next_coh_on_grant = co.masterMetadataOnGrant( c_gnt.payload, - xact_internal.meta.coh, + xact_meta.coh, c_gnt.header.dst) - val outer_write_acq = Bundle(UncachedWrite(xact.addr, UInt(trackerId), xact.data), + val outer_write_acq = Bundle(UncachedWrite(xact_addr, UInt(trackerId), xact_data(outer_data_write_cnt)), { case TLId => outerId }) - val outer_write_wb = Bundle(UncachedWrite(addr_wb, UInt(trackerId), wb_buffer), + val outer_write_wb = Bundle(UncachedWrite(wb_addr, UInt(trackerId), wb_buffer(outer_data_write_cnt)), { case TLId => outerId }) - val outer_read = Bundle(UncachedRead( xact.addr, UInt(trackerId)), { case TLId => outerId }) + val outer_read = Bundle(UncachedRead( xact_addr, UInt(trackerId)), { case TLId => outerId }) io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.payload := outer_read //default io.outer.acquire.bits.header.src := UInt(bankId) io.outer.grant.ready := Bool(true) //grant.data -> xact.data - val cprb_for_cacq = Probe(co.getProbeType(xact, xact_internal.meta.coh), xact.addr) - val cprb_for_mwb = Probe(co.getProbeTypeOnVoluntaryWriteback, addr_wb) - //TODO inner_probe_mprb + val cprb_for_cacq = Probe(co.getProbeType(xact, xact_meta.coh), xact_addr) + val cprb_for_mvwb = Probe(co.getProbeTypeOnVoluntaryWriteback, wb_addr) + //TODO cprb_for_mprb io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Mux(!tag_match && needs_writeback, - cprb_for_mwb, + io.inner.probe.bits.payload := Mux(!xact_tag_match && needs_writeback, + cprb_for_mvwb, cprb_for_cacq) - val cgnt_for_cacq = Grant(xact.uncached, co.getGrantType(xact, xact_internal.meta.coh), - xact.client_xact_id, + val cgnt_for_cacq = Grant(xact_uncached, co.getGrantType(xact, xact_meta.coh), + xact_client_xact_id, UInt(trackerId), - xact.data) - val cgnt_for_cwb = Grant(Bool(false), crel_wb_g_type, UInt(0), UInt(trackerId), UInt(0)) + xact_data(cgnt_data_cnt)) + val cgnt_for_cvwb = Grant(Bool(false), crel_wb_g_type, UInt(0), UInt(trackerId), UInt(0)) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := Mux(crel_was_voluntary, crel_wb_src, init_client_id) - io.inner.grant.bits.payload := Mux(crel_was_voluntary, cgnt_for_cwb, cgnt_for_cacq) + io.inner.grant.bits.header.dst := Mux(crel_was_voluntary, crel_wb_src, xact_src) + io.inner.grant.bits.payload := Mux(crel_was_voluntary, cgnt_for_cvwb, cgnt_for_cacq) io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) @@ -527,31 +585,45 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.data.read.valid := Bool(false) io.data.read.bits.id := UInt(trackerId) - io.data.read.bits.way_en := xact_internal.way_en - io.data.read.bits.addr := xact.addr + io.data.read.bits.way_en := xact_way_en + io.data.read.bits.addr := Cat(xact_addr, local_data_read_cnt) io.data.write.valid := Bool(false) io.data.write.bits.id := UInt(trackerId) - io.data.write.bits.way_en := xact_internal.way_en - io.data.write.bits.addr := xact.addr + io.data.write.bits.way_en := xact_way_en + io.data.write.bits.addr := Cat(xact_addr, local_data_write_cnt) io.data.write.bits.wmask := SInt(-1) - io.data.write.bits.data := xact.data + io.data.write.bits.data := xact_data(local_data_write_cnt) io.meta.read.valid := Bool(false) io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta.read.bits.tag := xact.addr >> UInt(untagBits) + io.meta.read.bits.idx := xact_addr(untagBits-1,blockOffBits) + io.meta.read.bits.tag := xact_addr >> UInt(untagBits) io.meta.write.valid := Bool(false) io.meta.write.bits.id := UInt(trackerId) - io.meta.write.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta.write.bits.way_en := xact_internal.way_en - io.meta.write.bits.data.tag := xact.addr >> UInt(untagBits) + io.meta.write.bits.idx := xact_addr(untagBits-1,blockOffBits) + io.meta.write.bits.way_en := xact_way_en + io.meta.write.bits.data.tag := xact_addr >> UInt(untagBits) io.meta.write.bits.data.coh := next_coh_on_grant + when(collect_cacq_data) { + io.inner.acquire.ready := Bool(true) + when(io.inner.acquire.valid) { + xact_data(cacq_data_cnt) := c_acq.payload.data + } + when(cacq_data_done) { collect_cacq_data := Bool(false) } + } + switch (state) { is(s_idle) { io.inner.acquire.ready := Bool(true) when( io.inner.acquire.valid ) { - xact := c_acq.payload - init_client_id := c_acq.header.src + xact_uncached := c_acq.payload.uncached + xact_a_type := c_acq.payload.a_type + xact_addr := c_acq.payload.addr + xact_client_xact_id := c_acq.payload.client_xact_id + xact_data(UInt(0)) := c_acq.payload.data + xact_subblock := c_acq.payload.subblock + xact_src := c_acq.header.src + collect_cacq_data := co.messageHasData(c_acq.payload) state := s_meta_read } } @@ -561,21 +633,23 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } is(s_meta_resp) { when(io.meta.resp.valid) { + xact_tag_match := io.meta.resp.bits.tag_match + xact_meta := io.meta.resp.bits.meta + xact_way_en := io.meta.resp.bits.way_en val coh = io.meta.resp.bits.meta.coh val _tag_match = io.meta.resp.bits.tag_match val _needs_writeback = !_tag_match && co.needsWriteback(coh) val _is_hit = _tag_match && co.isHit(xact, coh) val _needs_probes = co.requiresProbes(xact, coh) - xact_internal := io.meta.resp.bits when(_needs_probes) { val mask_incoherent = co.dir().full(coh.sharers) & ~io.tile_incoherent val mask_self = mask_incoherent & - ~(!(co.requiresSelfProbe(xact) || _needs_writeback) << init_client_id) + ~(!(co.requiresSelfProbe(xact) || _needs_writeback) << xact_src) pending_probes := mask_self release_count := co.dir().count(mask_self) crel_had_data := Bool(false) crel_was_voluntary := Bool(false) - } + } state := Mux(_tag_match, Mux(_is_hit, Mux(_needs_probes, s_probe, s_data_read_hit), @@ -594,25 +668,41 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) when(io.inner.release.valid) { - xact_internal.meta.coh := next_coh_on_release + xact_meta.coh := next_coh_on_release + + // Handle released dirty data when(co.messageHasData(c_rel.payload)) { crel_had_data := Bool(true) - when(tag_match) { - xact.data := mergeData(xact, io.inner.release.bits.payload.data) - } .otherwise { - wb_buffer := io.inner.release.bits.payload.data + when(xact_tag_match) { // Hit, so merge new write with released data + //TODO make sure cacq data is actually present before merging + xact_data(crel_data_cnt) := mergeData(xact, + xact_data(crel_data_cnt), + io.inner.release.bits.payload.data) + } .otherwise { // Miss, we're voluntarily evicting this data + wb_buffer(crel_data_cnt) := io.inner.release.bits.payload.data } } + + // Voluntary releases don't count against the release countdown + // because we will get a further release ack from that client in + // response to our probe. We don't send a grant acknowledging + // a writeback or decrement release_count until we've received + // all the data beats. when(co.isVoluntary(c_rel.payload)) { - crel_was_voluntary := Bool(true) - crel_wb_src := c_rel.header.src - crel_wb_g_type := co.getGrantTypeOnVoluntaryWriteback(xact_internal.meta.coh) - } - when(!co.isVoluntary(c_rel.payload)) { - release_count := release_count - UInt(1) + when(!co.messageHasData(c_rel.payload) || crel_data_done) { + crel_was_voluntary := Bool(true) + crel_wb_src := c_rel.header.src + crel_wb_g_type := co.getGrantTypeOnVoluntaryWriteback(xact_meta.coh) + } + } .otherwise { + when(!co.messageHasData(c_rel.payload) || crel_data_done) { + release_count := release_count - UInt(1) + } } } + // If we saw a voluntary writeback, we need to send an extra grant + // to acknowledge it. when(crel_was_voluntary) { io.inner.grant.valid := Bool(true) when(io.inner.grant.ready) { @@ -621,7 +711,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } when(release_count === UInt(0) && !crel_was_voluntary) { - state := Mux(tag_match, + state := Mux(xact_tag_match, Mux(is_hit, Mux(crel_had_data, s_data_write, s_data_read_hit), s_outer_read), @@ -630,20 +720,19 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } is(s_data_read_wb) { io.data.read.valid := Bool(true) - when(io.data.read.ready) { - state := s_data_resp_wb + when(local_data_read_done) { state := s_data_resp_wb } + when(io.data.resp.valid) { + wb_buffer(local_data_resp_cnt) := io.data.resp.bits.data } } is(s_data_resp_wb) { - when(io.data.resp.valid) { - wb_buffer := io.data.resp.bits.data - state := s_outer_write_wb - } + when(io.data.resp.valid) { wb_buffer(local_data_resp_cnt) := io.data.resp.bits.data } + when(local_data_resp_done) { state := s_outer_write_wb } } is(s_outer_write_wb) { io.outer.acquire.valid := Bool(true) io.outer.acquire.bits.payload := outer_write_wb - when(io.outer.acquire.ready) { + when(outer_data_write_done) { state := s_outer_read } } @@ -657,27 +746,35 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St is(s_outer_resp) { io.outer.grant.ready := Bool(true) when(io.outer.grant.valid) { - xact.data := mergeData(xact, io.outer.grant.bits.payload.data) - //TODO: set pending client state in xact_internal.meta.coh - state := Mux(co.messageHasData(io.outer.grant.bits.payload), - s_data_write, s_data_read_hit) + //TODO make sure cacq data is actually present before merging + xact_data(outer_data_resp_cnt) := mergeData(xact, xact_data(outer_data_resp_cnt), + io.outer.grant.bits.payload.data) + //TODO: set pending client state in xact_meta.coh + when(outer_data_resp_done) { + state := Mux(co.messageHasData(io.outer.grant.bits.payload), + s_data_write, s_data_read_hit) + } } } is(s_data_read_hit) { io.data.read.valid := Bool(true) - when(io.data.read.ready) { - state := s_data_resp_hit + when(io.data.resp.valid) { + //TODO make sure cacq data is actually present before merging + xact_data(local_data_resp_cnt) := mergeData(xact, xact_data(local_data_resp_cnt), + io.data.resp.bits.data) } + when(local_data_read_done) { state := s_data_resp_hit } } is(s_data_resp_hit) { when(io.data.resp.valid) { - xact.data := mergeData(xact, io.data.resp.bits.data) - state := s_meta_write + xact_data(local_data_resp_cnt) := mergeData(xact, xact_data(local_data_resp_cnt), + io.data.resp.bits.data) } + when(local_data_resp_done) { state := s_meta_write } } is(s_data_write) { io.data.write.valid := Bool(true) - when(io.data.write.ready) { + when(local_data_write_done) { state := s_meta_write } } @@ -687,7 +784,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } is(s_grant) { io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { + when(!co.messageHasData(c_gnt.payload) || cgnt_data_done) { state := Mux(co.requiresAckForGrant(c_gnt.payload), s_busy, s_idle) } diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 280e20ad..9d6d768b 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -205,9 +205,7 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(di def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) val uncached = MasterMetadata(masterValid, m.sharers)(this) - MuxBundle(uncached, Array( - g.is(grantReadExclusive) -> cached - )) + Mux(g.uncached, uncached, cached) } def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData @@ -339,9 +337,7 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(d def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) val uncached = MasterMetadata(masterValid, m.sharers)(this) - MuxBundle(uncached, Array( - g.is(grantReadExclusive) -> cached - )) + Mux(g.uncached, uncached, cached) } def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData @@ -485,10 +481,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(d def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) val uncached = MasterMetadata(masterValid, m.sharers)(this) - MuxBundle(uncached, Array( - g.is(grantReadShared) -> cached, - g.is(grantReadExclusive) -> cached - )) + Mux(g.uncached, uncached, cached) } def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData @@ -644,11 +637,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy( def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) val uncached = MasterMetadata(masterValid, m.sharers)(this) - MuxBundle(uncached, Array( - g.is(grantReadShared) -> cached, - g.is(grantReadExclusive) -> cached, - g.is(grantReadExclusiveAck) -> cached - )) + Mux(g.uncached, uncached, cached) } def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData @@ -817,12 +806,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) val uncached = MasterMetadata(masterValid, m.sharers)(this) - MuxBundle(uncached, Array( - g.is(grantReadShared) -> cached, - g.is(grantReadExclusive) -> cached, - g.is(grantReadExclusiveAck) -> cached, - g.is(grantReadMigratory) -> cached - )) + Mux(g.uncached, uncached, cached) } diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 678efb6a..45c6fa59 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -78,16 +78,19 @@ object Acquire { def requiresOuterRead(a_type: UInt) = a_type != uncachedWrite def requiresOuterWrite(a_type: UInt) = a_type === uncachedWrite - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt): Acquire = { + def apply(uncached: Bool, a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt, subblock: UInt): Acquire = { val acq = new Acquire - acq.uncached := Bool(false) + acq.uncached := uncached acq.a_type := a_type acq.addr := addr acq.client_xact_id := client_xact_id acq.data := data - acq.subblock := UInt(0) + acq.subblock := subblock acq } + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt): Acquire = { + apply(Bool(false), a_type, addr, client_xact_id, data, UInt(0)) + } def apply(a_type: Bits, addr: UInt, client_xact_id: UInt): Acquire = { apply(a_type, addr, client_xact_id, UInt(0)) } diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 69b13800..fcdbdd9f 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -15,20 +15,10 @@ abstract trait CoherenceAgentParameters extends UsesParameters val nTransactors = nReleaseTransactors + nAcquireTransactors val nClients = params(NClients) val sdqDepth = params(L2StoreDataQueueDepth)*tlDataBeats - val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(params(L2StoreDataQueueDepth))) + - log2Ceil(tlDataBeats) + val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(sdqDepth)) val nDataQueueLocations = 3 //Stores, VoluntaryWBs, Releases } -abstract class CoherenceAgent(innerId: String, outerId: String) extends Module - with CoherenceAgentParameters { - val io = new Bundle { - val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip - val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) - val incoherent = Vec.fill(nClients){Bool()}.asInput - } -} - class DataQueueLocation extends Bundle with CoherenceAgentParameters { val idx = UInt(width = dqIdxBits) val loc = UInt(width = log2Ceil(nDataQueueLocations)) @@ -42,6 +32,15 @@ object DataQueueLocation { } } +abstract class CoherenceAgent(innerId: String, outerId: String) extends Module + with CoherenceAgentParameters { + val io = new Bundle { + val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip + val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) + val incoherent = Vec.fill(nClients){Bool()}.asInput + } +} + class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends CoherenceAgent(innerId, outerId) { @@ -103,6 +102,12 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends } release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases + // Wire finished transaction acks + val ack = io.inner.finish + trackerList.map(_.io.inner.finish.valid := ack.valid) + trackerList.map(_.io.inner.finish.bits := ack.bits) + ack.ready := Bool(true) + // Wire probe requests to clients val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) io.inner.probe <> probe_arb.io.out @@ -115,12 +120,6 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends io.inner.grant <> grant_arb.io.out grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } - // Wire finished transaction acks - val ack = io.inner.finish - trackerList.map(_.io.inner.finish.valid := ack.valid) - trackerList.map(_.io.inner.finish.bits := ack.bits) - ack.ready := Bool(true) - // Create an arbiter for the one memory port val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), { case TLId => outerId; case TLDataBits => internalDataBits }) @@ -164,9 +163,13 @@ abstract class XactTracker(innerId: String, outerId: String) extends Module { class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { val s_idle :: s_outer :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) - val xact = Reg{ new Release } - val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) - val data_ptrs = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) } + + val xact_src = Reg(io.inner.release.bits.header.src.clone) + val xact_r_type = Reg(io.inner.release.bits.payload.r_type) + val xact_addr = Reg(io.inner.release.bits.payload.addr.clone) + val xact_client_xact_id = Reg(io.inner.release.bits.payload.client_xact_id.clone) + val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) } + val collect_inner_data = Reg(init=Bool(false)) val (inner_data_cnt, inner_data_done) = Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) @@ -174,33 +177,35 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && + io.has_release_conflict := co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && (state != s_idle) io.outer.grant.ready := Bool(false) io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits.header.src := UInt(bankId) - //io.outer.acquire.bits.header.dst TODO - io.outer.acquire.bits.payload := Bundle(UncachedWrite( - xact.addr, - UInt(trackerId), - data_ptrs(outer_data_cnt)), - { case TLId => outerId }) io.inner.acquire.ready := Bool(false) io.inner.probe.valid := Bool(false) io.inner.release.ready := Bool(false) io.inner.grant.valid := Bool(false) + io.inner.finish.ready := Bool(false) + io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := init_client_id + io.inner.grant.bits.header.dst := xact_src io.inner.grant.bits.payload := Grant(Bool(false), co.getGrantTypeOnVoluntaryWriteback(co.masterMetadataOnFlush), - xact.client_xact_id, + xact_client_xact_id, UInt(trackerId)) + io.outer.acquire.bits.header.src := UInt(bankId) + io.outer.acquire.bits.payload := Bundle(UncachedWrite( + xact_addr, + UInt(trackerId), + xact_data(outer_data_cnt)), + { case TLId => outerId }) + when(collect_inner_data) { io.inner.release.ready := Bool(true) when(io.inner.release.valid) { - data_ptrs(inner_data_cnt) := c_rel.payload.data + xact_data(inner_data_cnt) := c_rel.payload.data } when(inner_data_done) { collect_inner_data := Bool(false) } } @@ -209,15 +214,18 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute is(s_idle) { io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { - xact := c_rel.payload - init_client_id := c_rel.header.src - data_ptrs(UInt(0)) := c_rel.payload.data + xact_src := c_rel.header.src + xact_r_type := c_rel.payload.r_type + xact_addr := c_rel.payload.addr + xact_client_xact_id := c_rel.payload.client_xact_id + xact_data(UInt(0)) := c_rel.payload.data collect_inner_data := co.messageHasData(c_rel.payload) state := Mux(co.messageHasData(c_rel.payload), s_outer, s_ack) } } is(s_outer) { - io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) + io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) + else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) when(outer_data_done) { state := s_ack } } is(s_ack) { @@ -230,28 +238,28 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) val state = Reg(init=s_idle) - val xact = Reg{ new Acquire } - val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) - //TODO: Will need id reg for merged release xacts - val data_ptrs = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } + + val xact_src = Reg(io.inner.acquire.bits.header.src.clone) + val xact_uncached = Reg(io.inner.acquire.bits.payload.uncached.clone) + val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone) + val xact_addr = Reg(io.inner.acquire.bits.payload.addr.clone) + val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone) + val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone) + val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } + val xact = Acquire(xact_uncached, xact_a_type, xact_addr, xact_client_xact_id, UInt(0), xact_subblock) + val collect_inner_data = Reg(init=Bool(false)) val (inner_data_cnt, inner_data_done) = Counter(io.inner.acquire.fire() && co.messageHasData(io.inner.acquire.bits.payload), tlDataBeats) val (outer_data_cnt, outer_data_done) = Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) - val release_count = if (nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(nClients))) + val release_count = Reg(init=UInt(0, width = log2Up(nClients))) val probe_flags = Reg(init=Bits(0, width = nClients)) val curr_p_id = PriorityEncoder(probe_flags) val pending_outer_write = co.messageHasData(xact) val pending_outer_read = co.requiresOuterRead(xact, co.masterMetadataOnFlush) - val outer_write_acq = Bundle(UncachedWrite(xact.addr, UInt(trackerId), data_ptrs(outer_data_cnt)), - { case TLId => outerId }) - val outer_write_rel = Bundle(UncachedWrite(xact.addr, UInt(trackerId), c_rel.payload.data), - { case TLId => outerId }) - val outer_read = Bundle(UncachedRead(xact.addr, UInt(trackerId)), - { case TLId => outerId }) val probe_initial_flags = Bits(width = nClients) probe_initial_flags := Bits(0) @@ -260,26 +268,34 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) - io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) && !collect_inner_data - io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) + io.has_acquire_conflict := co.isCoherenceConflict(xact_addr, c_acq.payload.addr) && + (state != s_idle) && + !collect_inner_data + io.has_release_conflict := co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && + (state != s_idle) + + val outer_write_acq = Bundle(UncachedWrite(xact_addr, UInt(trackerId), xact_data(outer_data_cnt)), + { case TLId => outerId }) + val outer_write_rel = Bundle(UncachedWrite(xact_addr, UInt(trackerId), c_rel.payload.data), + { case TLId => outerId }) + val outer_read = Bundle(UncachedRead(xact_addr, UInt(trackerId)), { case TLId => outerId }) io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.payload := outer_read //default io.outer.acquire.bits.header.src := UInt(bankId) - //io.outer.acquire.bits.header.dst TODO - io.outer.acquire.bits.payload := outer_read io.outer.grant.ready := io.inner.grant.ready io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.masterMetadataOnFlush), xact.addr) + io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.masterMetadataOnFlush), xact_addr) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := init_client_id - io.inner.grant.bits.payload := Grant(xact.uncached, + io.inner.grant.bits.header.dst := xact_src + io.inner.grant.bits.payload := Grant(xact_uncached, co.getGrantType(xact, co.masterMetadataOnFlush), - xact.client_xact_id, + xact_client_xact_id, UInt(trackerId), UInt(0)) // Data bypassed in parent @@ -289,7 +305,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri when(collect_inner_data) { io.inner.acquire.ready := Bool(true) when(io.inner.acquire.valid) { - data_ptrs(inner_data_cnt) := c_acq.payload.data + xact_data(inner_data_cnt) := c_acq.payload.data } when(inner_data_done) { collect_inner_data := Bool(false) } } @@ -300,9 +316,13 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val needs_outer_write = co.messageHasData(c_acq.payload) val needs_outer_read = co.requiresOuterRead(c_acq.payload, co.masterMetadataOnFlush) when( io.inner.acquire.valid ) { - xact := c_acq.payload - init_client_id := c_acq.header.src - data_ptrs(UInt(0)) := c_acq.payload.data + xact_uncached := c_acq.payload.uncached + xact_a_type := c_acq.payload.a_type + xact_addr := c_acq.payload.addr + xact_client_xact_id := c_acq.payload.client_xact_id + xact_data(UInt(0)) := c_acq.payload.data + xact_subblock := c_acq.payload.subblock + xact_src := c_acq.header.src collect_inner_data := co.messageHasData(c_acq.payload) probe_flags := probe_initial_flags release_count := PopCount(probe_initial_flags) @@ -319,12 +339,12 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri } // Handle releases, which may have data to be written back + io.inner.release.ready := !co.messageHasData(c_rel.payload) || io.outer.acquire.ready when(io.inner.release.valid) { when(co.messageHasData(c_rel.payload)) { io.outer.acquire.valid := Bool(true) io.outer.acquire.bits.payload := outer_write_rel when(io.outer.acquire.ready) { - io.inner.release.ready := Bool(true) when(outer_data_done) { release_count := release_count - UInt(1) when(release_count === UInt(1)) { @@ -334,7 +354,6 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri } } } .otherwise { - io.inner.release.ready := Bool(true) release_count := release_count - UInt(1) when(release_count === UInt(1)) { state := Mux(pending_outer_write, s_mem_write, @@ -351,7 +370,8 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri } } is(s_mem_write) { - io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) + io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) + else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) io.outer.acquire.bits.payload := outer_write_acq when(outer_data_done) { state := Mux(pending_outer_read, s_mem_read, s_make_grant) From 6a8b66231ca484cd40269a8aa8170c0ef318b5fe Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 12 Dec 2014 12:07:04 -0800 Subject: [PATCH 267/374] Add uncached->cached tilelink converter --- uncore/src/main/scala/tilelink.scala | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 45c6fa59..4e5ac961 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -226,6 +226,25 @@ class TileLinkIO extends UncachedTileLinkIO { val release = new DecoupledIO(new LogicalNetworkIO(new Release)) } +class TileLinkIOWrapper extends TLModule { + val io = new Bundle { + val in = new UncachedTileLinkIO().flip + val out = new TileLinkIO + } + io.out.acquire <> io.in.acquire + io.out.grant <> io.in.grant + io.out.finish <> io.in.finish + io.out.probe.ready := Bool(false) + io.out.release.valid := Bool(false) +} +object TileLinkIOWrapper { + def apply[T <: Data](uncached: UncachedTileLinkIO) = { + val conv = Module(new TileLinkIOWrapper) + conv.io.in <> uncached + conv.io.out + } +} + abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule { type MasterSourcedWithId = MasterSourcedMessage with HasClientTransactionId type ClientSourcedWithId = ClientSourcedMessage with HasClientTransactionId From d04da83f96b58c7957483059ea7508d4d38925cd Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 15 Dec 2014 17:36:17 -0800 Subject: [PATCH 268/374] Make data RAMs 1RW instead of 1R1W --- uncore/src/main/scala/cache.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 97a0c9a5..09d04521 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -216,10 +216,13 @@ class L2DataArray extends L2HellaCacheModule { val wmask = FillInterleaved(8, io.write.bits.wmask) val resp = (0 until nWays).map { w => val array = Mem(Bits(width=rowBits), nSets*refillCycles, seqRead = true) + val reg_raddr = Reg(UInt()) when (io.write.bits.way_en(w) && io.write.valid) { array.write(waddr, io.write.bits.data, wmask) + }.elsewhen (io.read.bits.way_en(w) && io.read.valid) { + reg_raddr := raddr } - array(RegEnable(raddr, io.read.bits.way_en(w) && io.read.valid)) + array(reg_raddr) } io.resp.valid := ShiftRegister(io.read.fire(), 1) io.resp.bits.id := ShiftRegister(io.read.bits.id, 1) From ab39cbb15dbe4068f96882d7ab1923ad51c524e9 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 15 Dec 2014 19:23:13 -0800 Subject: [PATCH 269/374] cleanup DirectoryRepresentation and coherence params --- uncore/src/main/scala/cache.scala | 14 +-- uncore/src/main/scala/coherence.scala | 145 ++++++++++++-------------- uncore/src/main/scala/uncore.scala | 17 ++- 3 files changed, 86 insertions(+), 90 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 09d04521..08533ef9 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -123,7 +123,7 @@ object L2Metadata { } } class L2Metadata extends Metadata with L2HellaCacheParameters { - val coh = new MasterMetadata()(co) //co.masterMetadataOnFlush.clone + val coh = new MasterMetadata } class L2MetaReadReq extends MetaReadReq with HasL2Id { @@ -524,8 +524,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val (local_data_resp_cnt, local_data_resp_done) = Counter(io.data.resp.valid, tlDataBeats) val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) - val pending_probes = Reg(init = co.dir().flush) - val curr_p_id = co.dir().next(pending_probes) + val pending_probes = Reg(init = co.dir.flush) + val curr_p_id = co.dir.next(pending_probes) val needs_writeback = !xact_tag_match && co.needsWriteback(xact_meta.coh) val is_hit = xact_tag_match && co.isHit(xact, xact_meta.coh) @@ -645,11 +645,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val _is_hit = _tag_match && co.isHit(xact, coh) val _needs_probes = co.requiresProbes(xact, coh) when(_needs_probes) { - val mask_incoherent = co.dir().full(coh.sharers) & ~io.tile_incoherent + val mask_incoherent = co.dir.full(coh.sharers) & ~io.tile_incoherent val mask_self = mask_incoherent & ~(!(co.requiresSelfProbe(xact) || _needs_writeback) << xact_src) pending_probes := mask_self - release_count := co.dir().count(mask_self) + release_count := co.dir.count(mask_self) crel_had_data := Bool(false) crel_was_voluntary := Bool(false) } @@ -663,9 +663,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } } is(s_probe) { - io.inner.probe.valid := !co.dir().none(pending_probes) + io.inner.probe.valid := !co.dir.none(pending_probes) when(io.inner.probe.ready) { - pending_probes := co.dir().pop(pending_probes, curr_p_id) + pending_probes := co.dir.pop(pending_probes, curr_p_id) } // Handle releases, which may have data being written back diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 9d6d768b..d7bbd25f 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -3,46 +3,35 @@ package uncore import Chisel._ -abstract class CoherenceMetadata extends Bundle +abstract class CoherenceMetadata extends Bundle with CoherenceAgentParameters +class ClientMetadata extends CoherenceMetadata { + val state = UInt(width = co.clientStateWidth) + def ===(right: ClientMetadata): Bool = this.state === right.state + override def clone = new ClientMetadata().asInstanceOf[this.type] +} object ClientMetadata { - def apply(state: UInt)(implicit c: CoherencePolicy) = { + def apply(state: UInt) = { val m = new ClientMetadata m.state := state m } } -class ClientMetadata(implicit c: CoherencePolicy) extends CoherenceMetadata { - val state = UInt(width = c.clientStateWidth) - def ===(right: ClientMetadata): Bool = this.state === right.state - override def clone = new ClientMetadata()(c).asInstanceOf[this.type] -} +class MasterMetadata extends CoherenceMetadata { + val state = UInt(width = co.masterStateWidth) + val sharers = UInt(width = co.dir.width) + override def clone = new MasterMetadata().asInstanceOf[this.type] +} object MasterMetadata { - def apply(state: UInt)(implicit c: CoherencePolicy): MasterMetadata = { + def apply(state: UInt, sharers: UInt) = { val m = new MasterMetadata m.state := state - m.sharers := c.dir().flush - m - } - def apply(state: UInt, sharers: UInt)(implicit c: CoherencePolicy): MasterMetadata = { - val m = apply(state) m.sharers := sharers m } + def apply(state: UInt): MasterMetadata = apply(state, new MasterMetadata().co.dir.flush) } -class MasterMetadata(implicit c: CoherencePolicy) extends CoherenceMetadata { - val state = UInt(width = c.masterStateWidth) - val sharers = UInt(width = c.dir().width) - override def clone = new MasterMetadata()(c).asInstanceOf[this.type] -} -/* -class MixedMetadata(inner: CoherencePolicy, outer: CoherencePolicy) extends CoherenceMetadata { - val cstate = UInt(width = outer.clientStateWidth) - val mstate = UInt(width = inner.masterStateWidth) - val sharers = inner.dir.sharers.clone -} -*/ abstract class DirectoryRepresentation(val width: Int) { def pop(prev: UInt, id: UInt): UInt @@ -77,7 +66,7 @@ class FullRepresentation(nClients: Int) extends DirectoryRepresentation(nClients def full(s: UInt) = s } -abstract class CoherencePolicy(val dir: () => DirectoryRepresentation) { +abstract class CoherencePolicy(val dir: DirectoryRepresentation) { def nClientStates: Int def nMasterStates: Int def nAcquireTypes: Int @@ -143,7 +132,7 @@ abstract class CoherencePolicy(val dir: () => DirectoryRepresentation) { } } -class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { +class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 2 def nMasterStates = 2 def nAcquireTypes = 1 @@ -184,18 +173,18 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(di MuxLookup(cmd, clientInvalid, Array( M_INV -> clientInvalid, M_CLN -> clientValid - )))(this) + ))) def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = - ClientMetadata(Mux(incoming.uncached, clientInvalid, clientValid))(this) + ClientMetadata(Mux(incoming.uncached, clientInvalid, clientValid)) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, probeCopy -> m.state - )))(this) - def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) + ))) + def masterMetadataOnFlush = MasterMetadata(masterInvalid) def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, dir().pop(m.sharers, src))(this) + val next = MasterMetadata(masterValid, dir.pop(m.sharers, src)) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -203,8 +192,8 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(di )) } def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) - val uncached = MasterMetadata(masterValid, m.sharers)(this) + val cached = MasterMetadata(masterValid, dir.push(m.sharers, dst)) + val uncached = MasterMetadata(masterValid, m.sharers) Mux(g.uncached, uncached, cached) } @@ -263,11 +252,11 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(di def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir().none(m.sharers) + def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { +class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 3 def nMasterStates = 2 def nAcquireTypes = 2 @@ -307,27 +296,27 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(d def needsWriteback(m: MasterMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = - ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state))(this) + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state)) def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( MuxLookup(cmd, clientInvalid, Array( M_INV -> clientInvalid, M_CLN -> clientExclusiveClean - )))(this) + ))) def clientMetadataOnFlush() = clientMetadataOnCacheControl(M_INV) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( Mux(incoming.uncached, clientInvalid, Mux(outstanding.a_type === acquireReadExclusiveDirty, clientExclusiveDirty, - clientExclusiveClean)))(this) + clientExclusiveClean))) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, probeDowngrade -> clientExclusiveClean, probeCopy -> m.state - )))(this) - def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) + ))) + def masterMetadataOnFlush = MasterMetadata(masterInvalid) def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, dir().pop(m.sharers,src))(this) + val next = MasterMetadata(masterValid, dir.pop(m.sharers,src)) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -335,8 +324,8 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(d )) } def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) - val uncached = MasterMetadata(masterValid, m.sharers)(this) + val cached = MasterMetadata(masterValid, dir.push(m.sharers, dst)) + val uncached = MasterMetadata(masterValid, m.sharers) Mux(g.uncached, uncached, cached) } @@ -401,12 +390,12 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(d def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir().none(m.sharers) + def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { +class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 3 def nMasterStates = 2 def nAcquireTypes = 2 @@ -452,26 +441,26 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(d def needsWriteback(m: MasterMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = - ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state))(this) + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state)) def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( MuxLookup(cmd, clientInvalid, Array( M_INV -> clientInvalid, M_CLN -> clientShared - )))(this) + ))) def clientMetadataOnFlush() = clientMetadataOnCacheControl(M_INV) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( Mux(incoming.uncached, clientInvalid, Mux(incoming.g_type === grantReadShared, clientShared, - clientExclusiveDirty)))(this) + clientExclusiveDirty))) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, probeDowngrade -> clientShared, probeCopy -> m.state - )))(this) - def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) + ))) + def masterMetadataOnFlush = MasterMetadata(masterInvalid) def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, dir().pop(m.sharers,src))(this) + val next = MasterMetadata(masterValid, dir.pop(m.sharers,src)) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -479,8 +468,8 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(d )) } def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) - val uncached = MasterMetadata(masterValid, m.sharers)(this) + val cached = MasterMetadata(masterValid, dir.push(m.sharers, dst)) + val uncached = MasterMetadata(masterValid, m.sharers) Mux(g.uncached, uncached, cached) } @@ -525,7 +514,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(d def getGrantType(a: Acquire, m: MasterMetadata): UInt = { Mux(a.uncached, getGrantTypeForUncached(a, m), Mux(a.a_type === acquireReadShared, - Mux(!dir().none(m.sharers), grantReadShared, grantReadExclusive), + Mux(!dir.none(m.sharers), grantReadShared, grantReadExclusive), grantReadExclusive)) } def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck @@ -552,12 +541,12 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(d def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir().none(m.sharers) + def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { +class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 4 def nMasterStates = 2 def nAcquireTypes = 2 @@ -603,13 +592,13 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy( def needsWriteback(m: MasterMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = - ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state))(this) + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state)) def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( MuxLookup(cmd, clientInvalid, Array( M_INV -> clientInvalid, M_CLN -> clientShared - )))(this) + ))) def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( Mux(incoming.uncached, clientInvalid, @@ -618,16 +607,16 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy( grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusive, clientExclusiveDirty, clientExclusiveClean), grantReadExclusiveAck -> clientExclusiveDirty - ))))(this) + )))) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, probeDowngrade -> clientShared, probeCopy -> m.state - )))(this) - def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) + ))) + def masterMetadataOnFlush = MasterMetadata(masterInvalid) def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, dir().pop(m.sharers,src))(this) + val next = MasterMetadata(masterValid, dir.pop(m.sharers,src)) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -635,8 +624,8 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy( )) } def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) - val uncached = MasterMetadata(masterValid, m.sharers)(this) + val cached = MasterMetadata(masterValid, dir.push(m.sharers, dst)) + val uncached = MasterMetadata(masterValid, m.sharers) Mux(g.uncached, uncached, cached) } @@ -681,7 +670,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy( def getGrantType(a: Acquire, m: MasterMetadata): UInt = { Mux(a.uncached, getGrantTypeForUncached(a, m), Mux(a.a_type === acquireReadShared, - Mux(!dir().none(m.sharers), grantReadShared, grantReadExclusive), + Mux(!dir.none(m.sharers), grantReadShared, grantReadExclusive), grantReadExclusive)) } def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck @@ -708,12 +697,12 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy( def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir().none(m.sharers) + def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { +class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 7 def nMasterStates = 2 def nAcquireTypes = 3 @@ -760,12 +749,12 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata( Mux(isWrite(cmd), MuxLookup(m.state, clientExclusiveDirty, Array( clientExclusiveClean -> clientExclusiveDirty, - clientMigratoryClean -> clientMigratoryDirty)), m.state))(this) + clientMigratoryClean -> clientMigratoryDirty)), m.state)) def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( MuxLookup(cmd, clientInvalid, Array( M_INV -> clientInvalid, M_CLN -> clientShared - )))(this) + ))) def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( Mux(incoming.uncached, clientInvalid, @@ -779,7 +768,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo acquireInvalidateOthers -> clientMigratoryDirty, acquireReadExclusive -> clientMigratoryDirty, acquireReadShared -> clientMigratoryClean)) - ))))(this) + )))) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, @@ -791,10 +780,10 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo clientSharedByTwo -> clientShared, clientMigratoryClean -> clientSharedByTwo, clientMigratoryDirty -> clientInvalid)) - )))(this) - def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) + ))) + def masterMetadataOnFlush = MasterMetadata(masterInvalid) def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, dir().pop(m.sharers,src))(this) + val next = MasterMetadata(masterValid, dir.pop(m.sharers,src)) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -804,8 +793,8 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo )) } def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) - val uncached = MasterMetadata(masterValid, m.sharers)(this) + val cached = MasterMetadata(masterValid, dir.push(m.sharers, dst)) + val uncached = MasterMetadata(masterValid, m.sharers) Mux(g.uncached, uncached, cached) } @@ -853,7 +842,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def getGrantType(a: Acquire, m: MasterMetadata): UInt = { Mux(a.uncached, getGrantTypeForUncached(a, m), MuxLookup(a.a_type, grantReadShared, Array( - acquireReadShared -> Mux(!dir().none(m.sharers), grantReadShared, grantReadExclusive), + acquireReadShared -> Mux(!dir.none(m.sharers), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI for broadcast? ))) @@ -883,7 +872,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir().none(m.sharers) + def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index fcdbdd9f..411d9517 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -4,16 +4,22 @@ package uncore import Chisel._ case object NReleaseTransactors extends Field[Int] +case object NProbeTransactors extends Field[Int] case object NAcquireTransactors extends Field[Int] +case object NIncoherentClients extends Field[Int] +case object NCoherentClients extends Field[Int] case object L2StoreDataQueueDepth extends Field[Int] -case object NClients extends Field[Int] +case object L2CoherencePolicy extends Field[DirectoryRepresentation => CoherencePolicy] +case object L2DirectoryRepresentation extends Field[DirectoryRepresentation] abstract trait CoherenceAgentParameters extends UsesParameters with TileLinkParameters { val nReleaseTransactors = 1 val nAcquireTransactors = params(NAcquireTransactors) val nTransactors = nReleaseTransactors + nAcquireTransactors - val nClients = params(NClients) + val nCoherentClients = params(NCoherentClients) + val nIncoherentClients = params(NIncoherentClients) + val nClients = nCoherentClients + nIncoherentClients val sdqDepth = params(L2StoreDataQueueDepth)*tlDataBeats val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(sdqDepth)) val nDataQueueLocations = 3 //Stores, VoluntaryWBs, Releases @@ -41,7 +47,7 @@ abstract class CoherenceAgent(innerId: String, outerId: String) extends Module } } -class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends +class L2BroadcastHub(bankId: Int, innerId: String, outerId: String) extends CoherenceAgent(innerId, outerId) { val internalDataBits = new DataQueueLocation().getWidth @@ -143,11 +149,12 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends abstract class XactTracker(innerId: String, outerId: String) extends Module { - val (co, nClients, tlDataBeats) = (params(TLCoherence),params(NClients),params(TLDataBeats)) + val (co, tlDataBeats) = (params(TLCoherence), params(TLDataBeats)) + val nClients = params(NCoherentClients) + params(NIncoherentClients) val io = new Bundle { val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) - val tile_incoherent = Bits(INPUT, params(NClients)) + val tile_incoherent = Bits(INPUT, nClients) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) } From bfcfc3fe18ba88c45fec946324ac3e8bcff1c417 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 17 Dec 2014 14:28:14 -0800 Subject: [PATCH 270/374] refactor cache params --- uncore/src/main/scala/cache.scala | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 08533ef9..a63fafff 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -30,8 +30,6 @@ abstract trait CacheParameters extends UsesParameters { val rowWords = rowBits/wordBits val rowBytes = rowBits/8 val rowOffBits = log2Up(rowBytes) - val refillCyclesPerBeat = params(TLDataBits)/rowBits - val refillCycles = refillCyclesPerBeat*params(TLDataBeats) } abstract class CacheBundle extends Bundle with CacheParameters @@ -99,7 +97,12 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { } abstract trait L2HellaCacheParameters extends CacheParameters - with CoherenceAgentParameters + with CoherenceAgentParameters { + val idxMSB = idxBits-1 + val idxLSB = 0 + val refillCyclesPerBeat = params(TLDataBits)/rowBits + val refillCycles = refillCyclesPerBeat*params(TLDataBeats) +} abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters @@ -415,13 +418,13 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.data.write.bits.data := xact_data(local_data_cnt) io.meta.read.valid := Bool(false) io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact_addr(untagBits-1,blockOffBits) - io.meta.read.bits.tag := xact_addr >> UInt(untagBits) + io.meta.read.bits.idx := xact_addr(idxMSB,idxLSB) + io.meta.read.bits.tag := xact_addr >> UInt(idxBits) io.meta.write.valid := Bool(false) io.meta.write.bits.id := UInt(trackerId) - io.meta.write.bits.idx := xact_addr(untagBits-1,blockOffBits) + io.meta.write.bits.idx := xact_addr(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en - io.meta.write.bits.data.tag := xact_addr >> UInt(untagBits) + io.meta.write.bits.data.tag := xact_addr >> UInt(idxBits) io.meta.write.bits.data.coh := co.masterMetadataOnRelease(xact, xact_meta.coh, xact_src) @@ -505,7 +508,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val crel_wb_src = Reg(init = UInt(0, width = log2Up(nClients))) val crel_wb_g_type = Reg(init = UInt(0, width = co.grantTypeWidth)) val wb_buffer = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } - val wb_addr = Cat(xact_meta.tag, xact_addr(untagBits-1,blockOffBits)) + val wb_addr = Cat(xact_meta.tag, xact_addr(idxMSB,idxLSB)) val collect_cacq_data = Reg(init=Bool(false)) //TODO: zero width wires @@ -535,7 +538,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St //TODO: Are there any races between lines with the same idx? //TODO: Allow hit under miss for stores io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && - xact.addr(untagBits-1,0) === c_acq.payload.addr(untagBits-1,0) && + xact.addr(idxMSB,idxLSB) === c_acq.payload.addr(idxMSB,idxLSB) && (state != s_idle) && !collect_cacq_data io.has_release_conflict := (co.isCoherenceConflict(xact.addr, c_rel.payload.addr) || @@ -598,13 +601,13 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.data.write.bits.data := xact_data(local_data_write_cnt) io.meta.read.valid := Bool(false) io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact_addr(untagBits-1,blockOffBits) - io.meta.read.bits.tag := xact_addr >> UInt(untagBits) + io.meta.read.bits.idx := xact_addr(idxMSB,idxLSB) + io.meta.read.bits.tag := xact_addr >> UInt(idxBits) io.meta.write.valid := Bool(false) io.meta.write.bits.id := UInt(trackerId) - io.meta.write.bits.idx := xact_addr(untagBits-1,blockOffBits) + io.meta.write.bits.idx := xact_addr(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en - io.meta.write.bits.data.tag := xact_addr >> UInt(untagBits) + io.meta.write.bits.data.tag := xact_addr >> UInt(idxBits) io.meta.write.bits.data.coh := next_coh_on_grant when(collect_cacq_data) { From d121af7f942e3d1f25084e9d3a7000ef3f03fd77 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 18 Dec 2014 17:12:29 -0800 Subject: [PATCH 271/374] Simplify release handling --- uncore/src/main/scala/cache.scala | 22 +++++++++------------- uncore/src/main/scala/uncore.scala | 17 +++++++---------- 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index a63fafff..e3a8db22 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -303,17 +303,13 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac // Handle releases, which might be voluntary and might have data val release = io.inner.release - val voluntary = co.isVoluntary(release.bits.payload) - val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) - val block_releases = Bool(false) - val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b} - val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), conflict_idx) + val release_idx = Vec(trackerList.map(_.io.has_release_match)).indexWhere{b: Bool => b} for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.inner t.release.bits := release.bits - t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases + t.release.valid := release.valid && (release_idx === UInt(i)) } - release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases + release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) // Wire finished transaction acks val finish = io.inner.finish @@ -353,7 +349,7 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) val tile_incoherent = Bits(INPUT, nClients) val has_acquire_conflict = Bool(OUTPUT) - val has_release_conflict = Bool(OUTPUT) + val has_release_match = Bool(OUTPUT) val data = new L2DataRWIO val meta = new L2MetaRWIO } @@ -391,8 +387,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou Counter(io.data.write.fire(), tlDataBeats) io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && - (state != s_idle) + io.has_release_match := co.isVoluntary(c_rel.payload) io.outer.grant.ready := Bool(false) io.outer.acquire.valid := Bool(false) @@ -541,9 +536,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St xact.addr(idxMSB,idxLSB) === c_acq.payload.addr(idxMSB,idxLSB) && (state != s_idle) && !collect_cacq_data - io.has_release_conflict := (co.isCoherenceConflict(xact.addr, c_rel.payload.addr) || - co.isCoherenceConflict(wb_addr, c_rel.payload.addr)) && - (state != s_idle) + io.has_release_match := !co.isVoluntary(c_rel.payload) && + (co.isCoherenceConflict(xact.addr, c_rel.payload.addr) || + co.isCoherenceConflict(wb_addr, c_rel.payload.addr)) && + (state === s_probe) val next_coh_on_release = co.masterMetadataOnRelease( c_rel.payload, diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 411d9517..d98bf980 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -94,19 +94,16 @@ class L2BroadcastHub(bankId: Int, innerId: String, outerId: String) extends when(vwbdq_enq) { vwbdq(rel_data_cnt) := release.bits.payload.data } // Handle releases, which might be voluntary and might have data - val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) - val block_releases = Bool(false) - val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b} - val release_idx = Mux(voluntary, UInt(0), conflict_idx) + val release_idx = Vec(trackerList.map(_.io.has_release_match)).indexWhere{b: Bool => b} for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.inner t.release.bits := release.bits t.release.bits.payload.data := (if (i < nReleaseTransactors) DataQueueLocation(rel_data_cnt, inVolWBQueue) else DataQueueLocation(UInt(0), inClientReleaseQueue)).toBits - t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases + t.release.valid := release.valid && (release_idx === UInt(i)) } - release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases + release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) // Wire finished transaction acks val ack = io.inner.finish @@ -156,7 +153,7 @@ abstract class XactTracker(innerId: String, outerId: String) extends Module { val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) val tile_incoherent = Bits(INPUT, nClients) val has_acquire_conflict = Bool(OUTPUT) - val has_release_conflict = Bool(OUTPUT) + val has_release_match = Bool(OUTPUT) } val c_acq = io.inner.acquire.bits @@ -184,8 +181,7 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && - (state != s_idle) + io.has_release_match := co.isVoluntary(c_rel.payload) io.outer.grant.ready := Bool(false) io.outer.acquire.valid := Bool(false) @@ -278,7 +274,8 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri io.has_acquire_conflict := co.isCoherenceConflict(xact_addr, c_acq.payload.addr) && (state != s_idle) && !collect_inner_data - io.has_release_conflict := co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && + io.has_release_match := co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && + !co.isVoluntary(c_rel.payload) && (state != s_idle) val outer_write_acq = Bundle(UncachedWrite(xact_addr, UInt(trackerId), xact_data(outer_data_cnt)), From f234fe65ce4eb88d09fd10c0707a6bcf2d6f1269 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 19 Dec 2014 03:03:53 -0800 Subject: [PATCH 272/374] Initial verison of L2WritebackUnit, passes MiT2 bmark tests --- uncore/src/main/scala/cache.scala | 377 ++++++++++++++++---------- uncore/src/main/scala/coherence.scala | 21 +- 2 files changed, 253 insertions(+), 145 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index e3a8db22..3cbc8967 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -108,7 +108,7 @@ abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters trait HasL2Id extends Bundle with CoherenceAgentParameters { - val id = UInt(width = log2Up(nTransactors)) + val id = UInt(width = log2Up(nTransactors + 1)) } trait HasL2InternalRequestState extends L2HellaCacheBundle { @@ -284,32 +284,36 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac Module(new L2AcquireTracker(id, bankId, innerId, outerId)) } + val wb = Module(new L2WritebackUnit(nTransactors, bankId, innerId, outerId)) + doOutputArbitration(wb.io.wb.req, trackerList.map(_.io.wb.req)) + doInputRouting(wb.io.wb.resp, trackerList.map(_.io.wb.resp)) + // Propagate incoherence flags - trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) + (trackerList.map(_.io.tile_incoherent) :+ wb.io.tile_incoherent).map( _ := io.incoherent.toBits) // Handle acquire transaction initiation val acquire = io.inner.acquire - val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - val block_acquires = any_acquire_conflict + val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) - for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.inner - alloc_arb.io.in(i).valid := t.acquire.ready - t.acquire.bits := acquire.bits - t.acquire.valid := alloc_arb.io.in(i).ready + val acquireList = trackerList.map(_.io.inner.acquire) + acquireList zip alloc_arb.io.in map { case(acq, arb) => + arb.valid := acq.ready + acq.bits := acquire.bits + acq.valid := arb.ready } - acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && !block_acquires + acquire.ready := acquireList.map(_.ready).reduce(_||_) && !block_acquires alloc_arb.io.out.ready := acquire.valid && !block_acquires - // Handle releases, which might be voluntary and might have data + // Wire releases from clients val release = io.inner.release - val release_idx = Vec(trackerList.map(_.io.has_release_match)).indexWhere{b: Bool => b} - for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.inner - t.release.bits := release.bits - t.release.valid := release.valid && (release_idx === UInt(i)) + val release_idx = Vec(trackerList.map(_.io.has_release_match) :+ + wb.io.has_release_match).indexWhere{b: Bool => b} + val releaseList = trackerList.map(_.io.inner.release) :+ wb.io.inner.release + releaseList.zipWithIndex.map { case(r, i) => + r.bits := release.bits + r.valid := release.valid && (release_idx === UInt(i)) } - release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) + release.ready := Vec(releaseList.map(_.ready)).read(release_idx) // Wire finished transaction acks val finish = io.inner.finish @@ -321,27 +325,174 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac finish.ready := Vec(trackerList.map(_.io.inner.finish.ready)).read(finish_idx) // Wire probe requests to clients - doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) + doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe) :+ wb.io.inner.probe) // Wire grant reply to initiating client def hasData(m: LogicalNetworkIO[Grant]) = co.messageHasData(m.payload) doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant), tlDataBeats, hasData _) // Create an arbiter for the one memory port - val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), + val outerList = trackerList.map(_.io.outer) :+ wb.io.outer + val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(outerList.size), {case TLId => outerId}) - outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } + outerList zip outer_arb.io.in map { case(out, arb) => out <> arb } io.outer <> outer_arb.io.out // Wire local memories doOutputArbitration(io.meta.read, trackerList.map(_.io.meta.read)) doOutputArbitration(io.meta.write, trackerList.map(_.io.meta.write)) - doOutputArbitration(io.data.read, trackerList.map(_.io.data.read)) - doOutputArbitration(io.data.write, trackerList.map(_.io.data.write)) + doOutputArbitration(io.data.read, trackerList.map(_.io.data.read) :+ wb.io.data.read, tlDataBeats) + doOutputArbitration(io.data.write, trackerList.map(_.io.data.write), tlDataBeats) doInputRouting(io.meta.resp, trackerList.map(_.io.meta.resp)) - doInputRouting(io.data.resp, trackerList.map(_.io.data.resp)) + doInputRouting(io.data.resp, trackerList.map(_.io.data.resp) :+ wb.io.data.resp) } +class L2WritebackReq extends L2HellaCacheBundle + with HasL2Id { + val addr = UInt(width = tlAddrBits) + val coh = new MasterMetadata + val way_en = Bits(width = nWays) +} + +class L2WritebackResp extends L2HellaCacheBundle with HasL2Id + +class L2WritebackIO extends L2HellaCacheBundle { + val req = Decoupled(new L2WritebackReq) + val resp = Valid(new L2WritebackResp).flip +} + +class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2HellaCacheModule { + val io = new Bundle { + val wb = new L2WritebackIO().flip + val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip + val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) + val tile_incoherent = Bits(INPUT, nClients) + val has_acquire_conflict = Bool(OUTPUT) + val has_release_match = Bool(OUTPUT) + val data = new L2DataRWIO + } + val c_acq = io.inner.acquire.bits + val c_rel = io.inner.release.bits + val c_gnt = io.inner.grant.bits + val c_ack = io.inner.finish.bits + val m_gnt = io.outer.grant.bits + + val s_idle :: s_probe :: s_data_read :: s_data_resp :: s_outer_write :: Nil = Enum(UInt(), 5) + val state = Reg(init=s_idle) + + val xact_addr = Reg(io.inner.acquire.bits.payload.addr.clone) + val xact_coh = Reg{ new MasterMetadata } + val xact_way_en = Reg{ Bits(width = nWays) } + val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } + val xact_id = Reg{ UInt() } + + val crel_had_data = Reg(init = Bool(false)) + val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) + val pending_probes = Reg(init = co.dir.flush) + val curr_p_id = co.dir.next(pending_probes) + + val (crel_data_cnt, crel_data_done) = + Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) + val (outer_data_write_cnt, outer_data_write_done) = + Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) + val (local_data_read_cnt, local_data_read_done) = Counter(io.data.read.fire(), tlDataBeats) + val (local_data_resp_cnt, local_data_resp_done) = Counter(io.data.resp.valid, tlDataBeats) + + io.has_release_match := !co.isVoluntary(c_rel.payload) && + co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && + (state === s_probe) + + val next_coh_on_rel = co.masterMetadataOnRelease(c_rel.payload, xact_coh, c_rel.header.src) + + io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.payload := Bundle(UncachedWrite(xact_addr, + UInt(trackerId), + xact_data(outer_data_write_cnt)), + { case TLId => outerId }) + io.outer.acquire.bits.header.src := UInt(bankId) + io.outer.grant.ready := Bool(false) // Never gets mgnts + + io.inner.probe.valid := Bool(false) + io.inner.probe.bits.header.src := UInt(bankId) + io.inner.probe.bits.header.dst := curr_p_id + io.inner.probe.bits.payload := Probe(co.getProbeTypeOnVoluntaryWriteback, xact_addr) + + io.inner.grant.valid := Bool(false) + io.inner.acquire.ready := Bool(false) + io.inner.release.ready := Bool(false) + io.inner.finish.ready := Bool(false) + + io.data.read.valid := Bool(false) + io.data.read.bits.id := UInt(trackerId) + io.data.read.bits.way_en := xact_way_en + io.data.read.bits.addr := Cat(xact_addr, local_data_read_cnt) + io.data.write.valid := Bool(false) + + io.wb.req.ready := Bool(false) + io.wb.resp.valid := Bool(false) + io.wb.resp.bits.id := xact_id + + switch (state) { + is(s_idle) { + io.wb.req.ready := Bool(true) + when(io.wb.req.valid) { + xact_addr := io.wb.req.bits.addr + xact_coh := io.wb.req.bits.coh + xact_way_en := io.wb.req.bits.way_en + xact_id := io.wb.req.bits.id + val coh = io.wb.req.bits.coh + val needs_probes = co.requiresProbesOnVoluntaryWriteback(coh) + when(needs_probes) { + val mask_incoherent = co.dir.full(coh.sharers) & ~io.tile_incoherent + pending_probes := mask_incoherent + release_count := co.dir.count(mask_incoherent) + crel_had_data := Bool(false) + } + state := Mux(needs_probes, s_probe, s_data_read) + } + } + is(s_probe) { + // Send probes + io.inner.probe.valid := !co.dir.none(pending_probes) + when(io.inner.probe.ready) { + pending_probes := co.dir.pop(pending_probes, curr_p_id) + } + // Handle releases, which may have data being written back + io.inner.release.ready := Bool(true) + when(io.inner.release.valid) { + xact_coh := next_coh_on_rel + // Handle released dirty data + when(co.messageHasData(c_rel.payload)) { + crel_had_data := Bool(true) + xact_data(crel_data_cnt) := c_rel.payload.data + } + // We don't decrement release_count until we've received all the data beats. + when(!co.messageHasData(c_rel.payload) || crel_data_done) { + release_count := release_count - UInt(1) + } + } + when(release_count === UInt(0)) { + state := Mux(crel_had_data, s_outer_write, s_data_read) + } + } + is(s_data_read) { + io.data.read.valid := Bool(true) + when(io.data.resp.valid) { xact_data(local_data_resp_cnt) := io.data.resp.bits.data } + when(local_data_read_done) { state := s_data_resp } + } + is(s_data_resp) { + when(io.data.resp.valid) { xact_data(local_data_resp_cnt) := io.data.resp.bits.data } + when(local_data_resp_done) { state := s_outer_write } + } + is(s_outer_write) { + io.outer.acquire.valid := Bool(true) + when(outer_data_write_done) { + io.wb.resp.valid := Bool(true) + state := s_idle + } + } + } +} abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCacheModule { val io = new Bundle { @@ -352,6 +503,7 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa val has_release_match = Bool(OUTPUT) val data = new L2DataRWIO val meta = new L2MetaRWIO + val wb = new L2WritebackIO } val c_acq = io.inner.acquire.bits @@ -423,6 +575,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.meta.write.bits.data.coh := co.masterMetadataOnRelease(xact, xact_meta.coh, xact_src) + io.wb.req.valid := Bool(false) when(collect_inner_data) { io.inner.release.ready := Bool(true) @@ -482,8 +635,9 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou } } + class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_meta_read :: s_meta_resp :: s_probe :: s_data_read_wb :: s_data_resp_wb :: s_outer_write_wb :: s_outer_read :: s_outer_resp :: s_data_read_hit :: s_data_resp_hit :: s_data_write :: s_outer_write_acq :: s_meta_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 16) + val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_probe :: s_outer_read :: s_outer_resp :: s_data_read :: s_data_resp :: s_data_write :: s_meta_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 14) val state = Reg(init=s_idle) val xact_src = Reg(io.inner.acquire.bits.header.src.clone) @@ -499,32 +653,31 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val xact = Acquire(xact_uncached, xact_a_type, xact_addr, xact_client_xact_id, UInt(0), xact_subblock) val crel_had_data = Reg(init = Bool(false)) - val crel_was_voluntary = Reg(init = Bool(false)) - val crel_wb_src = Reg(init = UInt(0, width = log2Up(nClients))) - val crel_wb_g_type = Reg(init = UInt(0, width = co.grantTypeWidth)) - val wb_buffer = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } - val wb_addr = Cat(xact_meta.tag, xact_addr(idxMSB,idxLSB)) + val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) + val pending_probes = Reg(init = UInt(0, width = nCoherentClients)) + val curr_p_id = co.dir.next(pending_probes) + val full_sharers = co.dir.full(io.meta.resp.bits.meta.coh.sharers) + val mask_self = Mux(co.requiresSelfProbe(xact), + full_sharers | (UInt(1) << xact_src), + full_sharers & ~UInt(UInt(1) << xact_src, width = nClients)) + val mask_incoherent = mask_self & ~io.tile_incoherent val collect_cacq_data = Reg(init=Bool(false)) //TODO: zero width wires - val (cacq_data_cnt, cacq_data_done) = + val (cacq_data_cnt, cacq_data_done) = Counter(io.inner.acquire.fire() && co.messageHasData(io.inner.acquire.bits.payload), tlDataBeats) - val (crel_data_cnt, crel_data_done) = + val (crel_data_cnt, crel_data_done) = Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) - val (cgnt_data_cnt, cgnt_data_done) = + val (cgnt_data_cnt, cgnt_data_done) = Counter(io.inner.grant.fire() && co.messageHasData(io.inner.grant.bits.payload), tlDataBeats) - val (outer_data_write_cnt, outer_data_write_done) = + val (outer_data_write_cnt, outer_data_write_done) = Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) - val (outer_data_resp_cnt, outer_data_resp_done) = + val (outer_data_resp_cnt, outer_data_resp_done) = Counter(io.outer.grant.fire() && co.messageHasData(io.outer.grant.bits.payload), tlDataBeats) val (local_data_read_cnt, local_data_read_done) = Counter(io.data.read.fire(), tlDataBeats) val (local_data_write_cnt, local_data_write_done) = Counter(io.data.write.fire(), tlDataBeats) val (local_data_resp_cnt, local_data_resp_done) = Counter(io.data.resp.valid, tlDataBeats) - val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) - val pending_probes = Reg(init = co.dir.flush) - val curr_p_id = co.dir.next(pending_probes) - val needs_writeback = !xact_tag_match && co.needsWriteback(xact_meta.coh) val is_hit = xact_tag_match && co.isHit(xact, xact_meta.coh) val needs_probes = co.requiresProbes(xact, xact_meta.coh) @@ -537,22 +690,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St (state != s_idle) && !collect_cacq_data io.has_release_match := !co.isVoluntary(c_rel.payload) && - (co.isCoherenceConflict(xact.addr, c_rel.payload.addr) || - co.isCoherenceConflict(wb_addr, c_rel.payload.addr)) && + co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state === s_probe) - val next_coh_on_release = co.masterMetadataOnRelease( - c_rel.payload, - xact_meta.coh, - c_rel.header.src) - val next_coh_on_grant = co.masterMetadataOnGrant( - c_gnt.payload, - xact_meta.coh, + val next_coh_on_rel = co.masterMetadataOnRelease(c_rel.payload, xact_meta.coh, c_rel.header.src) + val next_coh_on_gnt = co.masterMetadataOnGrant(c_gnt.payload, xact_meta.coh, c_gnt.header.dst) - val outer_write_acq = Bundle(UncachedWrite(xact_addr, UInt(trackerId), xact_data(outer_data_write_cnt)), - { case TLId => outerId }) - val outer_write_wb = Bundle(UncachedWrite(wb_addr, UInt(trackerId), wb_buffer(outer_data_write_cnt)), + val outer_write = Bundle(UncachedWrite(xact_addr, UInt(trackerId), xact_data(outer_data_write_cnt)), { case TLId => outerId }) val outer_read = Bundle(UncachedRead( xact_addr, UInt(trackerId)), { case TLId => outerId }) @@ -561,25 +706,18 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.outer.acquire.bits.header.src := UInt(bankId) io.outer.grant.ready := Bool(true) //grant.data -> xact.data - val cprb_for_cacq = Probe(co.getProbeType(xact, xact_meta.coh), xact_addr) - val cprb_for_mvwb = Probe(co.getProbeTypeOnVoluntaryWriteback, wb_addr) - //TODO cprb_for_mprb io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Mux(!xact_tag_match && needs_writeback, - cprb_for_mvwb, - cprb_for_cacq) + io.inner.probe.bits.payload := Probe(co.getProbeType(xact, xact_meta.coh), xact_addr) - val cgnt_for_cacq = Grant(xact_uncached, co.getGrantType(xact, xact_meta.coh), - xact_client_xact_id, - UInt(trackerId), - xact_data(cgnt_data_cnt)) - val cgnt_for_cvwb = Grant(Bool(false), crel_wb_g_type, UInt(0), UInt(trackerId), UInt(0)) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := Mux(crel_was_voluntary, crel_wb_src, xact_src) - io.inner.grant.bits.payload := Mux(crel_was_voluntary, cgnt_for_cvwb, cgnt_for_cacq) + io.inner.grant.bits.header.dst := xact_src + io.inner.grant.bits.payload := Grant(xact_uncached, co.getGrantType(xact, xact_meta.coh), + xact_client_xact_id, + UInt(trackerId), + xact_data(cgnt_data_cnt)) io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) @@ -604,7 +742,13 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.meta.write.bits.idx := xact_addr(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en io.meta.write.bits.data.tag := xact_addr >> UInt(idxBits) - io.meta.write.bits.data.coh := next_coh_on_grant + io.meta.write.bits.data.coh := next_coh_on_gnt + + io.wb.req.valid := Bool(false) + io.wb.req.bits.addr := Cat(xact_meta.tag, xact_addr(idxMSB,idxLSB)) + io.wb.req.bits.coh := xact_meta.coh + io.wb.req.bits.way_en := xact_way_en + io.wb.req.bits.id := UInt(trackerId) when(collect_cacq_data) { io.inner.acquire.ready := Bool(true) @@ -641,101 +785,50 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val coh = io.meta.resp.bits.meta.coh val _tag_match = io.meta.resp.bits.tag_match val _needs_writeback = !_tag_match && co.needsWriteback(coh) + val _needs_probes = _tag_match && co.requiresProbes(xact, coh) val _is_hit = _tag_match && co.isHit(xact, coh) - val _needs_probes = co.requiresProbes(xact, coh) when(_needs_probes) { - val mask_incoherent = co.dir.full(coh.sharers) & ~io.tile_incoherent - val mask_self = mask_incoherent & - ~(!(co.requiresSelfProbe(xact) || _needs_writeback) << xact_src) - pending_probes := mask_self - release_count := co.dir.count(mask_self) + pending_probes := mask_incoherent(nCoherentClients-1,0) + release_count := co.dir.count(mask_incoherent) crel_had_data := Bool(false) - crel_was_voluntary := Bool(false) } state := Mux(_tag_match, - Mux(_is_hit, - Mux(_needs_probes, s_probe, s_data_read_hit), - Mux(_needs_probes, s_probe, s_outer_read)), - Mux(_needs_writeback, - Mux(_needs_probes, s_probe, s_data_read_wb), - s_outer_read)) + Mux(_needs_probes, s_probe, Mux(_is_hit, s_data_read, s_outer_read)), // Probe, hit or upgrade + Mux(_needs_writeback, s_wb_req, s_outer_read)) // Evict ifneedbe } } + is(s_wb_req) { + io.wb.req.valid := Bool(true) + when(io.wb.req.ready) { state := s_wb_resp } + } + is(s_wb_resp) { + when(io.wb.resp.valid) { state := s_outer_read } + } is(s_probe) { + // Send probes io.inner.probe.valid := !co.dir.none(pending_probes) when(io.inner.probe.ready) { pending_probes := co.dir.pop(pending_probes, curr_p_id) } - // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) when(io.inner.release.valid) { - xact_meta.coh := next_coh_on_release - + xact_meta.coh := next_coh_on_rel // Handle released dirty data when(co.messageHasData(c_rel.payload)) { crel_had_data := Bool(true) - when(xact_tag_match) { // Hit, so merge new write with released data - //TODO make sure cacq data is actually present before merging - xact_data(crel_data_cnt) := mergeData(xact, - xact_data(crel_data_cnt), - io.inner.release.bits.payload.data) - } .otherwise { // Miss, we're voluntarily evicting this data - wb_buffer(crel_data_cnt) := io.inner.release.bits.payload.data - } + //TODO make sure cacq data is actually present before merging + xact_data(crel_data_cnt) := mergeData(xact, + xact_data(crel_data_cnt), + c_rel.payload.data) } - - // Voluntary releases don't count against the release countdown - // because we will get a further release ack from that client in - // response to our probe. We don't send a grant acknowledging - // a writeback or decrement release_count until we've received - // all the data beats. - when(co.isVoluntary(c_rel.payload)) { - when(!co.messageHasData(c_rel.payload) || crel_data_done) { - crel_was_voluntary := Bool(true) - crel_wb_src := c_rel.header.src - crel_wb_g_type := co.getGrantTypeOnVoluntaryWriteback(xact_meta.coh) - } - } .otherwise { - when(!co.messageHasData(c_rel.payload) || crel_data_done) { - release_count := release_count - UInt(1) - } + // We don't decrement release_count until we've received all the data beats. + when(!co.messageHasData(c_rel.payload) || crel_data_done) { + release_count := release_count - UInt(1) } } - - // If we saw a voluntary writeback, we need to send an extra grant - // to acknowledge it. - when(crel_was_voluntary) { - io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { - crel_was_voluntary := Bool(false) - } - } - - when(release_count === UInt(0) && !crel_was_voluntary) { - state := Mux(xact_tag_match, - Mux(is_hit, - Mux(crel_had_data, s_data_write, s_data_read_hit), - s_outer_read), - Mux(crel_had_data, s_outer_write_wb, s_data_read_wb)) - } - } - is(s_data_read_wb) { - io.data.read.valid := Bool(true) - when(local_data_read_done) { state := s_data_resp_wb } - when(io.data.resp.valid) { - wb_buffer(local_data_resp_cnt) := io.data.resp.bits.data - } - } - is(s_data_resp_wb) { - when(io.data.resp.valid) { wb_buffer(local_data_resp_cnt) := io.data.resp.bits.data } - when(local_data_resp_done) { state := s_outer_write_wb } - } - is(s_outer_write_wb) { - io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_write_wb - when(outer_data_write_done) { - state := s_outer_read + when(release_count === UInt(0)) { + state := Mux(is_hit, Mux(crel_had_data, s_data_write, s_data_read), s_outer_read) } } is(s_outer_read) { @@ -754,20 +847,20 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St //TODO: set pending client state in xact_meta.coh when(outer_data_resp_done) { state := Mux(co.messageHasData(io.outer.grant.bits.payload), - s_data_write, s_data_read_hit) + s_data_write, s_data_read) } } } - is(s_data_read_hit) { + is(s_data_read) { io.data.read.valid := Bool(true) when(io.data.resp.valid) { //TODO make sure cacq data is actually present before merging xact_data(local_data_resp_cnt) := mergeData(xact, xact_data(local_data_resp_cnt), io.data.resp.bits.data) } - when(local_data_read_done) { state := s_data_resp_hit } + when(local_data_read_done) { state := s_data_resp } } - is(s_data_resp_hit) { + is(s_data_resp) { when(io.data.resp.valid) { xact_data(local_data_resp_cnt) := mergeData(xact, xact_data(local_data_resp_cnt), io.data.resp.bits.data) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index d7bbd25f..c7999f4b 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -117,6 +117,7 @@ abstract class CoherencePolicy(val dir: DirectoryRepresentation) { def requiresOuterWrite(acq: Acquire, m: MasterMetadata): Bool def requiresSelfProbe(a: Acquire): Bool def requiresProbes(a: Acquire, m: MasterMetadata): Bool + def requiresProbesOnVoluntaryWriteback(m: MasterMetadata): Bool def requiresAckForGrant(g: Grant): Bool def requiresAckForRelease(r: Release): Bool def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool @@ -253,6 +254,7 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) + def requiresProbesOnVoluntaryWriteback(m: MasterMetadata) = !dir.none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -391,6 +393,7 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) + def requiresProbesOnVoluntaryWriteback(m: MasterMetadata) = !dir.none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -541,7 +544,11 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) + def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) && + Mux(dir.one(m.sharers), Bool(true), + Mux(a.uncached, a.a_type != Acquire.uncachedRead, + a.a_type != acquireReadShared)) + def requiresProbesOnVoluntaryWriteback(m: MasterMetadata) = !dir.none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -697,7 +704,11 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) + def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) && + Mux(dir.one(m.sharers), Bool(true), + Mux(a.uncached, a.a_type != Acquire.uncachedRead, + a.a_type != acquireReadShared)) + def requiresProbesOnVoluntaryWriteback(m: MasterMetadata) = !dir.none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -872,7 +883,11 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) + def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) && + Mux(dir.one(m.sharers), Bool(true), + Mux(a.uncached, a.a_type != Acquire.uncachedRead, + a.a_type != acquireReadShared)) + def requiresProbesOnVoluntaryWriteback(m: MasterMetadata) = !dir.none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } From 2ef4357ca8492fd08e00a94692e95ab7bbdd82bd Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 19 Dec 2014 17:39:23 -0800 Subject: [PATCH 273/374] acquire allocation bugfix --- uncore/src/main/scala/cache.scala | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 3cbc8967..932392e3 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -293,14 +293,19 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac // Handle acquire transaction initiation val acquire = io.inner.acquire - val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) val acquireList = trackerList.map(_.io.inner.acquire) - acquireList zip alloc_arb.io.in map { case(acq, arb) => + val acquireMatchList = trackerList.map(_.io.has_acquire_match) + val any_acquire_matches = acquireMatchList.reduce(_||_) + val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} + val match_idx = Vec(acquireMatchList).indexWhere{b: Bool => b} + val acquire_idx = Mux(any_acquire_matches, match_idx, alloc_idx) + acquireList.zip(alloc_arb.io.in).zipWithIndex.map { case((acq, arb), i) => arb.valid := acq.ready acq.bits := acquire.bits - acq.valid := arb.ready + acq.valid := acquire.valid && (acquire_idx === UInt(i)) } + val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) acquire.ready := acquireList.map(_.ready).reduce(_||_) && !block_acquires alloc_arb.io.out.ready := acquire.valid && !block_acquires @@ -367,7 +372,6 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) val tile_incoherent = Bits(INPUT, nClients) - val has_acquire_conflict = Bool(OUTPUT) val has_release_match = Bool(OUTPUT) val data = new L2DataRWIO } @@ -500,6 +504,7 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) val tile_incoherent = Bits(INPUT, nClients) val has_acquire_conflict = Bool(OUTPUT) + val has_acquire_match = Bool(OUTPUT) val has_release_match = Bool(OUTPUT) val data = new L2DataRWIO val meta = new L2MetaRWIO @@ -539,6 +544,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou Counter(io.data.write.fire(), tlDataBeats) io.has_acquire_conflict := Bool(false) + io.has_acquire_match := Bool(false) io.has_release_match := co.isVoluntary(c_rel.payload) io.outer.grant.ready := Bool(false) @@ -689,8 +695,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St xact.addr(idxMSB,idxLSB) === c_acq.payload.addr(idxMSB,idxLSB) && (state != s_idle) && !collect_cacq_data + io.has_acquire_match := co.messageHasData(xact) && + (xact.addr === c_acq.payload.addr) && + collect_cacq_data io.has_release_match := !co.isVoluntary(c_rel.payload) && - co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && + (xact.addr === c_rel.payload.addr) && (state === s_probe) val next_coh_on_rel = co.masterMetadataOnRelease(c_rel.payload, xact_meta.coh, c_rel.header.src) From e62c71203ed2b0e2deca8d860c9e52ad26c94e5e Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 22 Dec 2014 18:50:37 -0800 Subject: [PATCH 274/374] disconnect unused outer network headers --- uncore/src/main/scala/cache.scala | 2 -- uncore/src/main/scala/uncore.scala | 2 -- 2 files changed, 4 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 932392e3..d871e652 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -413,7 +413,6 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str UInt(trackerId), xact_data(outer_data_write_cnt)), { case TLId => outerId }) - io.outer.acquire.bits.header.src := UInt(bankId) io.outer.grant.ready := Bool(false) // Never gets mgnts io.inner.probe.valid := Bool(false) @@ -712,7 +711,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.payload := outer_read //default - io.outer.acquire.bits.header.src := UInt(bankId) io.outer.grant.ready := Bool(true) //grant.data -> xact.data io.inner.probe.valid := Bool(false) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index d98bf980..e8e82d22 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -198,7 +198,6 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute xact_client_xact_id, UInt(trackerId)) - io.outer.acquire.bits.header.src := UInt(bankId) io.outer.acquire.bits.payload := Bundle(UncachedWrite( xact_addr, UInt(trackerId), @@ -286,7 +285,6 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.payload := outer_read //default - io.outer.acquire.bits.header.src := UInt(bankId) io.outer.grant.ready := io.inner.grant.ready io.inner.probe.valid := Bool(false) From c76b4bc21ddd1052195f593bf6071bc2e7151634 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 29 Dec 2014 22:55:18 -0800 Subject: [PATCH 275/374] TileLink doc --- uncore/doc/TileLink3.0Specification.pdf | Bin 0 -> 123869 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 uncore/doc/TileLink3.0Specification.pdf diff --git a/uncore/doc/TileLink3.0Specification.pdf b/uncore/doc/TileLink3.0Specification.pdf new file mode 100644 index 0000000000000000000000000000000000000000..363f9b03d037b340981ea2f27c752bbd2741a20c GIT binary patch literal 123869 zcmY!laBVf>Z^4=fsl4ocwey{eZ;u)M5og z1p|d3E;~D})VvgsIzuyb4Ip(O)kUd!B~a!1L8-<0rA5i9#R@hy`T<4x$-${5#R}2- z0WNL|`XQ+mB?|hUxryniPEgtzN_)m~>AO~xxCfUcmZXC8x*I4Mn!}u+@0Ooe0upsI zP%yLri|V@>Di~TqnMMjm24JS09mvJ{zNsmhiB9#pN`FULW!3z4JL7rS7LO~;=q@=(~U%$M(TrWL8KRqW^FF8L~ zzqBaFzR(sVr)un|YUHMB-EaoS~PRQmSg?R*;fbtZL-u>**Zi zZ(!MbiS5jqbs8(8>ZtLh4nC0x}9_mwGkd%~D9O73L zR32qi?35T$W*HygoD>?U2?`@SyI3xL=lr~q)Vz{n1tav>1`B|q-Nr`WEie@cm3r) z|K)4@|KA?h*FR9#Rr`1F{{Pp<>udh<*Zp`kHSvDE;*B4z_W%DqU;lOb{{LT(%m0(v z_wQHzo+BM^wcBoOI2M1;er0CMXLai{Ie!-4G@H@Ddo4brIlk`y$M*Y&!n@-4|9|Sf zXXEn!de1(5{e8dgPcg^N|EI6N|Nr-oQQOBZ<8!=AL>PshYcG{5pP=PXRB<-s7~iMa zEHBqfxZiSYW6*pYT74&+GqT~zC#m8oyi3#=FWu5Mdvti)MG*yC;a!`nrF%bkZGFnC zV#;WAIs3@Fhm)KtTQBlW&~Qkqy((5#vid=w*H2To*L*L7)~%0m`evrpdqMY|+tz(a zoV)j2DSZ)NZ{DIU+rE2+$AyQYQi(EgGyl2&m(p4Bs(t%~rp?E98L}GOo^;Ek^Z3eK z1+Og6ds>Bc>=T2cUu933S077NT=oiH8ONur)`(G)U2%jtR@1vt!~D|?$0a(hR)M`Y7shn8ygPID zw#L@*^H=Y3>{5SHyVyNry2Qo6jMax6yvysvc9qO|A(-aw@_Y7$`YB3LdZt&_9zHt9 z!{zx6$0ZA%f6J0p`qiMJsrK^K$~Ov=7Ei4Z^uF!Tb!UZ;e%9>YT0s{)H=2pfO&8US z&6nKj|7F_6bWzV;f=}I|++MoqES|K>{Bq}~8=b9J9#`7D_#bRD|NiQwne%!p;?(pm zXTFlT^i=t4vS?|r(~f6%945sqomAlZLRES?*Wd78Y>QmBTy#rr*Vyk>=d|;!K$P}H zGg0yL&1*ugF)Ms|%`2?=>`39pu1WhiCG8yRCU$Wf-Dz_c4U5TqCM&-5w!*XLT0z@Y zmFnkRbQJH>o}&=8eX97vJ@Y;4rmno;mdev!veIGU^OYC(+Kb)sov`cTC3C4wW@j^h zXKS)<*)+p%o6NeMyi5M97TNET#=CMquS*?QN%5zrzZx_Y-xbHMlkWIa?Up?=L{nX@2{k*Ad>+ImW4oe)4yLFUr+G8%@nf}X3`)@MyPZ!01|70c@ zdWbk4$=Q^&O*LJ*P3B83SLnzjNR8 zhq>QncGpbr>g0S z{XWZ6A?xE)YCjm=kEyMUev-~})Wo;#($)PESy5Vs=erkP4q@GKeP;BwzhU}^Pcddb z4SKU}=grGYZ{#i(b(7OByUi!F;bwp6rb{sllE)V=y=e8aOyWs+p28<>zAuexpS0gP zZ=QJlRmqCkH&SIzTAD1`G5x|^=j9L8N`-{K$JUoX`F-F5K55 z_C)H_;+!qs@)sA@Ww}g|;Jb86xZKLm*E;j+TVnw|o};%jCWLf_F0qBi5wk7Xy%U-a7VE4CLVlp`|-(d zuFe2c)EIWG^&UPxUO_e$O>znrPN?G4u`(1+fD_AjK zvs17-Y9N-~z1G@0&g)p@(ql_gwuLq=Gfr3R5@z#UbF5qT(Squk%&O;0GuPbM+w8qBUS{tiN%VZIOvg zZTyFJa{<+>g_WD6JGeGn4ewfVZ%(Mk@+-6W**xd>mUvk>UeQ_-@=5Wgm602}*J7(F z&D>V!T%S~)VD_y{ao-?(Q>3)znTM0odbK$Uw+`z(NqqKLdGbHoyQO=tJvvdRv|jDZ zJ=>DwD%;=x;Jdm|-Sf1?K4;IJK8p-azvav^TwyY%+XCn9^fer~$wR=6k1icf{b ze7W1J3bp5FBvX|Pcl}OL@}7Tgx`!J7%N;ZJx_acEF>F)j+Ih*2$tB5K?v_HyUgODu z!qw;Qxjk8A!l-2F5|uc`KxXZPz&Fko)+L%MTu%zr(-#W`EDN^XsB%+1_2tT)LY%jw z_)|5{{GKEfnVD&&6n~e0_bGczEz3VBx^|O=ki%FmV&RSTx zDImFesgbN$=q`mI&5O}3dL3O-CehKCct6j|tS}dhJ-1%T;fdRl|JD6`x!Fr=KNSj} z)L$Yr|8Zse3FXOQ3f;C=pF&QCX{hdYXPfQ*Rc|$y*?fg8pzg<&{aIu z^hSwir)c=|1;g z%6qriHM>o03!IlhUrRD~Pkb zVa=)+HK}g|mN?JX%AUODz)gQv)tWT3?~8YH-2KBV%m1x3%p>?&VSFQ36a%J8QJoAcRz%9FU(du@@i9uh`Y?q)QdVLzr)_2THW|( z`Ze2y&CUX!erkHXwm1~{>YZCB&+Q5c?o%7i7EJN0e$+42;-|vHr5HH%%mKsm{0>q# zmskds@p64ij#r(-)+*pL>14Qm?(>OX${z}!iLmjz?l(v7;$_*b@$aUcd|}n0^LdYm z>R_-W`>ZGUklYllV2=?}Jh&Tl-ppMlG9E(Ms*9<@plw)J(z!{DCf&U zW2MO!@1!-w*}}~BJxu&)b|^=eBlX6P2?DCy`rB34o~XTEEc*8FmaB1_8dsz^N8VMv z?Y2sd?^(_1dz}eWuU)i>dA8#c%QLMPSrY9!HLr|f9#&6w{osE>MzYUkV|`HDc8*xx z$-dE(|EzfG`&?<2pSW4(2Nv^1}>m=csV~r+hu{3u&MC=sHS^|4{$@BWmff^9Giz9X%foKRI(k<;Lm>Q5{TA#5tQd z>UGX_7P)CXoRFK^HQC%cy`)fc%SFybYrb}cIIZjRIQX|?Z`Gx_){+-h<>!QPa&Eft zNkdNWgZ&G!ViWd1{Q||G9MX0KahjES78ht8JhF{t!KD6t7q;EWd(Om9c4ZCA3(g&y?u7>DTnUb{WYp+ot3CaM1-_De=tD;TN8UBwu>g-hC*Ock(~hN0*wt zt#-M3Y_AqKh?#PcpLdB_mAXof*(9AW!Y()eJXyrz*Jq<}=~;itqMLKquQ>GNjInk4 zI*yRyM~72Psx?xrm_lEQX>JV7mw&SCoPMCxpOTATq-@P53xDA_ejB<;y>RNg znPPlz7k|>0t?UnW^jySqYkA0fp_R;R|3Bynmf00)oSS~bLm0UP-Lze2utYvnY?s+vdcdu+z=(nYzxqBN|%zC?HlF~UXmAKFbkMi7Y zS3e|gF1+f%+rLXwVEM(;oX1U9PFXmX@QUQNur&7OIW?_mf4Gp_TX?GbZ8!N(YIn{) zUK7MQU22sw&&q@QJr7%#xc?0*y7;zZ#q}3~;?vSB>_eU!9CGLlQ8fv+Y zoX+i?(0Q4)HRzm{;JI#fZpof`^E96DTwhwTiaB!P%~|Tx3alRO?6TtLIV;evTr3&( z$t9JfApu6(sQL!>Lu1zwBzOGI?y-?=S9bI$x0^>(_I(FXnnqJGt^8IZu z7nr9Mu5{?rLjkeO7wWCun<_s~b=~0UUUqwx{AS4juLw!r=Ps!_=BIA1I8@5{dtoZ8 zQME`x!<+8yTUOVdnf9)!!L>Y8`p!t};V>x2J$#JuRs*qr7b_o8_D>}B%H zS~MKjJuGUFn-ISw)92HnrSWFV!{byccvgIHINv+Py5WlFihHa9!jd;!^q$XlGYr)@ zxtk%tXX^2lB~LpQ^wm3VtvLAl^~IlWvYW4$H@sMKeCn}{mS-ykH+FsU?dZ*|I&`3O zqQvau$|t<%%Pp6F;@aRLZPPPVf78tqJ0}bA|5cLZpAyjbNMQTYr}0)Zo}AuavQ2Y( z{Y!;zrc;K?KD(NyTrBe8%Z*medAaT*-<1qyMkVcaJ2vV0w4d+Qe!Z#tw8Op^0q-|$ z*zWtJdz0Pf$(Adtysz)~6brsLb26`swZZh_i8{*)7PC*%mhSNC?zyUYHAVRJv-$%I z^pCsAd+k27v+EtJW;V}pHl@OYUtD`07WjoEe(q0JzG~uG@-j8vrL@^+-l2_-9@ih8 zo1LR@s(t4twR00sMrbO&Iq}MovGWRp%T*hrW3erP-Ob$UenIc8JEY1T)vXvp`q{Xm z3o>n<2Uboll>EGGO@i2!jlJ{oCq0$YvJFmr8}1(z@LhGn)uo9Wz7^KqJW)2~;T-vy zUb7o=iYufKX8CGX>9ismuaw`p zn{UbNsy2~{IdO|8h*3}g$KS^RA%qvucdwF?7hD1C&3rPO5QT~bscezeYHz(QJ(F(pIV!3KG^sP zyq$A^%U*PDNnt|Xvlp?SMW3lnENAY#Un(?V?aCD|Zq5Ghaoe%_I>XG~Q)ky4uQJ!l zR|s2NWURgNUhgUXj+-mqJSZ~PGT*%FbzVr`CaynewXA2v^q!Jda$CG< z^Vz#A?wnbq-xK|0rRG^3*~2PpM30+iADPdwC$izE%Y(OEf6H0!iL{6uTlM*VLEv2O z@a+x0)2B4u@(|_cUH5tWQ-;_RHYaaJvgDpvYWCj2#R3cNs%I{3RxY~JTGN#A_{*yB`(YcUrsn4C}T4+b_0RzsU7U zeOIn?D{tZs@AfzPPhzc7R-dxTUwp|;yv<+o-q#IY?dN(!b{EIn9^3Xvf%(CvxVKWZ z9o*6D%cNbK8l`P2A9~n#73RF!H+yF8E3qlfEsxW#Zgno#x%E!m;i!jQepskAOLFzD zlFabEea+lT+aK@9NQoA4|CN3J;?&|kF0Zq*cJqge%gXQN`*QwkQG2aIUU^#cg?0P; z!#n@6Dfz}+;p+^wyUMrmY=&3JSp#kVuS%bV$}=bUic6fDyf$I;tG$z|JS+5$9bB4O zGIe21d*0k@tWMiar|W5C^0!w7zGT|fbBl9P*v#@5*RIIwF37j4Ryz8Y_qmqV1^+_| zW|wL&U!QEWZBkaJnbrO((XVQE4Ou$`*f!mHs^uuRD~9*hxrHYUUS@p0XBOn@p>gij z@r9xF%vD?_p>z74FPK;q%fCD7udH}I*W{Ep-(VlQ76=WgoD{mk`FVXxH#eI{EByOQKIj_k&$#Vjm6y?V?2l5rdx72#o#V$?4}L7@ zn6mD_W_V;@?D2k`lQu8(b+6wpG_d@l_tsFgQJU+9ePI=!0nagV&1kV ziXJOf`xy&lSUR>uUktq9R+`di&9#qNvPe``@_l&cgu8(s*8MkBo&H>WuEMP+`=7br z%xc`rq$jv3TKv^j*Eb7pulpIur}zGs{PLFxXZwpbyuJQMgFPkY^WvPB&$tvfcD=In z$Wq?Pwd;j{&eZ!2ObaK-K5V;hP-XQbey5nmsX~p@%~LE~lcv8YetVvAQ7wOt@m~LV z+4X5%a+@*&R<^4y+tU_xO+lMo+~D`;TYR(wR zCT8BL722$gAjJ5TnQFXs#M@-W983wxhk*k9OwYG>CgFOJpOcP$0WS6r!8j+}Bo zKqG8(`D0U)jY|&Cn!K?8T-=Kv1sVIBUdX?k@S?2u?_14W1*3AIU#}8gq-~7uDeWzM zpY|};r;_*i`?Rg0KDDNm<(ofs&zA8Mu$^h4KRqYf)mLrboRxXggp}A1Po8fkU^=Tt zZ~fWGrLoFUyCVhJ>$};FEU&yx-kbGkLf(OgT}Pu++e}|vRlIxm;NC`#+=HoZ6+e#H z37+p)TKafFcYBidsztjZC!Lt;QSfh9Wd4q4M<4Yj2k)soBBG{Rx|zpd>gu_7)MTrJ zK^6Skw8H3;oxgwmaj`RvU#xytOvA=w(w+A@it_FAN}eCNqdsxgVFjP=r}Ijrqw|&? zn6iu4WoL9r3geHB~u`Bgju=(yZ zmb|Gij+_7A7|LY(l5bbWR0Ze5w>z#diHF~e=UATT^|8Zx{V_iFsgH#VI;cZGA& zpRS6hJ96$e`EjoMT;y=zg>dZM#|-!1U(q$^?L){n`&;{BEl@Y_OZtBn)36`Og5e>%VK%8hs{$CAb0r$}qsKPy`KX7#hTL7Go<(*vR| zMC`08_^AlL?iZEL-@p{hnH^ z?#cFB-K=X=w3XjIYKIwj=2Bi4sV^IT`U||ZTyRpQ;?f~Yu_i3uV-d@@896@ zc-tDyHQ&;wT@-lj93z}OH@3ne{QNzozpK_4zi4Au3b0AvJaaeOjqpvL*7@!y)@^*g zF>0xW`XkSEJ4@MP>uxJfd0ZU!G@xYBrB%xQt-qaTedjH_-mtpnf{8%jj3n*Wpoft> z{oQ}^n1htJeLA?CcaGG?uWm1|hp0!3O*5@;c#*h1{C)do!SM64^Y_SotFo<3oc!+Y z#+>{kTm5p{Z@2!a_{;eA#I_5eGvc1^`55B$Xj|>u-cO;bX`eGvb(if9+*_43*ZJ{Z z_7ZuE9S45DJ{!xL$-OPgS1z=oK4nxlT7 zxz~6r>qF@CLav#18Yb6y%;ld(Jnr1}bHkC{T_+rL!kw6iLo&=9ZLO`e1ck^NT>A$#yqO=6GzI{iWf-#lG&kY`()wAKkbZYI~~K zSM^pw;QQc`pw)@#4PE(Jy|nrp?Qf?->LOU0@b|NCM6$x zx{bf%)wzR5VnsM|6;>Twwa(%^^P(2fPRpd*YzyaPuUIv!Y{E17-aj9Cf2oVS{NZkO z{n+=p+582$m-aJ7oj3lQ{bBZ@SMA?_AM%>AGoCkbvTw%|%}-N)zs}pT zW!{&KPv88N=ywh*+VVqEHd;iu=+TPfvYU)*n0NkqEjM9B)vXWz3T}j-)Y59Y_2AVf z)|>i0CpTX_TOlDKK54I)Te?!(gItA85AEX)Yvo)jJi&ZgHhh@6d&e-U;0*UdgJ7Z^;sK%)7)H{%R%ZGhze$TU^{Ou&+MA``vFgd1%)Kuj zFX%qzy645?1-GBAo+>cce@W>f)$sE_4o2H-{{HG?#v|AHpS35o&$~Y5gTHu3DF3g6 zdF8S4yPv;Z$1a;Twd9CKN{W7aWc=?s?UEU-d*1I^zuAjVJNb9Ri>eFS?_aNBS}pqW zMxt2IetB-$G)4XMN5xiaJWuzXQmoOwP-caMY}!iG8u@jXN&;RMFU^@7`awrw(%Qpo zCOvcayeYCV(&V+*&AinscFJD&o?@o=VZn}yOUt-zR&;V!@O%83tbb(2l4Tw#$0dV* z%qmZGne$ZoMUY&EZc&N-)xY0^W}IM^lCZKryK82|ac+MnpU9hq9$bwHg2a=KT1Y{{%`y{sd8 z=*u*@m{SV_Cv#7EYn)u;Evj&dSv9y#IwDtXQDAW161jS>)k0O#uD$sSdrxE*zvw?c z|HtbYX&;4m8SGqtMJzgm@B4d)wkX$L4V6oYrWW(%Q?4{l~ZJFc0KWt<`j4u-2Zf@_383GMgm0@KdNEb-y#KCW<*b)>XZin^+hyErU(P?%Gk(kOVwS2|C(qs} z5Cu(ecR%@>vU~HXjz1~C*%X60k|)J7pM85}=0ux~&$hkpKQZmWwAq=YhF}D~*w1m;a4pv(he!ho<~x30yT{3zKxfxz2mxEwxG!!s;D;D}_pbifwe13Y?xV zxO>OC`MXtUx`$QS>4ELYx?<;^~qGiPk|CU(A=Q=7Ku?{pha zhnbT5pX^Osw9Eh1tlQb21q`qEdBsN0E3lt%eDa=NulUa=-kj(8&U{{Z^UCwz4=)rc z@#hef|0`(r^=F;K`Zitdj^%zEyQ*4}*XVIa&-qd3FunEcBB4XCHf=CJ#1bthylvs? zdv8v!K7HtBM@r*voo%5{bemO{GfWgb<|DmO@Ni@+&v8YEl!&Q&&bBO<%(?02u;o!E z+jEyEx!;`CPwdF%UozisuWr6?(7!)>nw{%{{@FaaW@$8?e~!YZN$PzBS!=yuxD*}f+?w==Kbqswo7()9YN{2kU88n0AkxQK!GKHO z#|un`gGgT=1rsj)U`K_LqS90@eJ5W9eZTyo+{7Hv;vbmtpruqO#)H;bAuWdj*=fL~ z@03`a3X)OK4=GAbDoxHvE%6QJ(s#{E&QHnAOIOhMOi9fv$t$l8YplI2L6l=Jp@U?Buo-QDrpg1++(s%ZB3CVZ&bn#6rK*S$dIJmO7BsJGFFD)OmtSu-tJ+ru^ zs8T`0F(p4KRg+8KAGA0Q6zv+YXx9WKnSz3x)LhU?Km$;`>W71t&qbRW7$|_Cv4NqU zk(s4wtb(zbLbR!crJjX}G05GfmKJ)3CMKo|W`@RkMiz#~u?i+83eiU97J9};2B2hO zpkQiXre|zo4hkNSS*E7udS-^^pujOuFtISuvotkVFtsqyGc_|aiB&MOP>41)GSxG- zG%!;DsWmjRuv9RyG}SW%X@ZNGnwaXD7#bKU7@Jw@nSs=p8W`$Xni?A_n3|aB8C#f{ zD;Qar>KU4vft+D#V4`PiX=18iWND&jX=G@rU}0>aXJBrDq}4bEBsexPA{`OkT;R2kZkai$MhZrt zToVKl@k`82Ma!RVIf?1Tpi;mQlp{+NER8MnEX_0DnYiT;e zCw+#D{FqxCR{2)Bd=_HZ_%Qm_q^G^Smi8&zn0Q>}KDJF;I(K#4{BLngea{u`w`|!V z-E&>*_=NB5E!rDye)&~;ox{Vv-(Xqg$ycs3@7~O3xG|mKhM8_wcHFPuDx0|yYNQ>0 zr9VF9_f@L$lV5}4YKDyH$Cuj7e6JnPWw4qPe~ZhftR?JxEc@#o*8dF=tT&#oY};$d z^Zw`cSy%t9`gi%1-%qieR~Ggx_a^3ldCII2|7-Q1)l=p&D<9-9-6U5N*1+>rfss{s z$zh#0*4lrCOI=+Y<{F;c%&}Ic-t~Q3kRkUZ0kaPGggsw0a+Fj9x{~$T6`7B(bNeH5 z=r2cp!f#d<^+Sp+g&f|3NdkEeGbCH2T8xgJzvo(|J-aH3~fFY(QuHB3{iQ25h&mpe@r>NW~BEk6V|cu1c3 zuG5lvuhC$ma3Z&pvB|@#_LqV6_Kz%g{L4S0^S?hyY^8wv`HXtWZrdzgg}M$7R8F55GCxHM;w;HvOQK-0~OOzvw7t@GEz# zKfAo6-Mu;6FKOTLTCJ9u|C3`L?Eas&>tR*GKJi283A48_R?evLu|LD^ERj6<=yBB_ z3mf8#e>%;0+4obqJF;iCLY~Y7&g2=#1rOUFGLhqdv0l+;!E(l0N4fL5GoOeV3g;zl zy7c~`uIAbOvBJ;q`_+HRpC0Faqq#ur_RoLo8s{XtYB)!*_+GM@vX%YoWd6O6_TG5T z=vrkoA^S-4hK}2_K9(z=X-j;xc!sc<^CUI4QvvyGzE&lV@91x~yAwS1gUz%3t{Wtu z@Al4d{GB+zLODk)R-pZxPV(bJxrJm-v) z-`e)XbLb!T_!;UkL3`qJolk#6E8jI2?At%xQ83@RhWn7gy2%!1)917p&gHcXvi+vG zp)R*%x z{eXp`*s=$5TCUd5p5)!R%)H`t)fw6P)~rP9lSK`Em_}k@)6U# z{_2+E02LdC$AYs>b~45m{`F~DDkc0S!%gY0E-w`>&8*;i6!pmIKkGKL z`O8I?T{yX7_sM_fUvmA=p0zt+#xkzTiS4H=_#XGJ?cVk1Q(My0Pd9ei#5GxnOt(MC zda&Z9!_7k7Gxqb7>+D;9cpf>g^w0QFvc&o9hYqJH&zDa7edO)J z88RJ?+P9SdU#i%nqnFQfEV;-)&RDeSJ@5VtW>2u$#s2BY<2Q3Y%ol!Bpl-c-ivB~fUOuS{aj`|B>-u*Z&EOYT zTH{g?Q6v9uf&779o(cC_cl6E`T=Ru9%r`8IKQYjh#gSROTc`1J%&&uvj#ErFExP)W zH}Rp{3a7$P=Tt#*H5)E;4@w*qS+s7zoX0}t@bo&;) zeOV_jrJlYWwxztJZ0(!4t-oHGX0PA*ZCB~_`vs4G&91w;`QfwP?RTnfFFPBZxBv0E z`2T6Ix9xuS@2&0kfA9AFmai!I?aXig=|$h%e%pUXCi~m|?Ma{h{pQ{M_cw1Zzh7Km z@vrr-{=P4tD)~#rER79QoAv2_C|xoyT0flTsc{IP&Aj zmLr#r{BkM3)a>c6BI{SB{qjm=RhXUs-@34chnaVLv#escZr7E){`gjNPX1089`WVf z+(&djzJ9d-_*UgB8KFt>H{88T;^KfKk?e!Xq;rOHR&fA44b z5PgvEfid%!_iw*{d!O{|ef-`1tM>1%PyIhdU+$06^NI_aefLXhUU$~Z?|*ruT>YF} zeCZ+Pf~3cVpF1b2UlIN|U;p2qx5A%3CeP3P`)2p~4PF0p*qxrIYul~-W zPrmi<|C@4CUC*rli&FW%&^>zRxfqroPq-5IZQ<>gi%Y)ySCz-zGd3u4XQ)d{k&Qjy zss2zgg5P}N#SNn84y;cQzQeS?`S^oG3wG&FCi6pc3&i#{-+my!q4C87;~!%0nvXwx z|H1ZN|F*-yAEabjmM1Ej91_cDx_*$i{ei(A*8Bs46~g;kTtDdT;eCI=_yW)Ww%ZSx zEBNoVi~W$U;Wa;;l+n66K~twO^#g-WyXXbBw-*>=FK}uvl-}obHima~B5w?H_Cj&F zme&uW_8g7jw|A1aaqpYd-!V_ZLxNMCN$}+HClWtfYP?hyDY_*or72B5f8zNQ3roj; zPWu$ZgdEqXOq!!o&no}pq-N2qIg0U<)}Oe4;`);Q8s~kg=_kvdRDUY}#Qc+e)-&f% z{-5+eDgQLLbU!zlTcyioa@`|kGa=`;2@yZIYaH7qS@ei0d{k)^@>=QOsujpPwbSd( zgT`2ui54rIwm#vSuR8_V#^`VAKIR(hffqta^p4WX$w|14K&_*fy1b&M`f{x z)z(WMQm%_v_8vPR@41Ya8|W%G%g9rMp9>IFGJn^c)%w^EhcLw2P) zZ;E23itiC$v59Gym~MM$NBXR5sy-pKX^M{WX;0}$$6KE4FI8ewR9~vCK9Tj|(B={e zS-$E7{XI_Vhw^^d?%^^&Ait3PS0>vX-unmUKM?$3yoW_!Sa%Nlo~72u&**HQ?yIXi zGbkp+qHcj{Az4@-NU zJ|63}l`Ec=K4b9-<2SD+o88%T+xSpm$Q#e%2PsdlGII;`Pf00SA^o3GW6$FEuf40K znrvsOT()nRnA6Rib?Bo2uZFZ&$M4OzXX)*`5neJS${^)L_OB44mlrz70>WabLr=I0oD%bR#WMq32Fq1bbea8>& zgV$}YIq_dHXS@GtiMY7o&MS|4zq&Fq-i+ycxvqA?Tl;<4Tcm#HnDw+tskN)G66l>2 zv*+T~Ehi=i$lq@Hx6`Ba=q=lGQX%oZho7!X3@e&pbGYwuQQ1DFdhYLKlI0yU-^Ta_ zg^2fxX3f~T!Fg^I|M4eGG6oBGR!oj8DQbLmC`K&laA=cE&#n}?91hRcr8yo6>&2!W zyOsT>@qE2vP)>lF=E=o8vrf0jG8?g8`{{a->5k8W#H@|GGbS`l_t9cyiGVDJ3N|$)IRs|09z<-H{iURW0x;wY<|O#r|u;=2J(%9V$|WOT&UsI*Xb21{-|Nxnps3q2JAj{Tmlv-=DY8 z+0@ePotFOn8(}f$r9_eS-=?N%Tf++fptz1Jr}wzG+pS~z;UqmLd%7Y z4FU@p7s@T~#hOH%4#Zj#2?+b1p*ymxAAtCW`T`g+ZH%A>W(8aH=0EL6$Ybq=q1 zZv5oKxnzxt*3%VtZ*Nn)`Z`JDpmqDgiuXBfkM?Yqka{)uWWbtoHfQHOS9y+BEH_U1 z9A+r;cJ8r&DcdJ4^vGv*4tqb9$F<_L9*nxi)L|jE#*F`Kq(8-{1dl z=2!i7_1Sy3?0oufT6$E=uWQ@Fc2&yVw3)u;tm5p|@77l>dv$cx_Ek&HUVcCS-U zZCq%$)$(>mt@_IM3c0QH6DTaP(i@jHLSW3vS%Ms{PIWwa9PlsH6Y3 zExK`klIZW*Qb+4tg5KnJbp0)L+$g`S>#yme8~lw)r^NW|&1sv;QwCD~tK})1UR9{g0Am z8+YH*Y^fD!P7u9(uA0FoV{t~_?o6f}XX8)Hx-P_klcJMd8oBl5S|HtP7 z_ty43=*=~6$!q?-@x8#EuJRSt8~!KpKZt!>crWje^cCI{8$~OfzSu^z^S@dynZk1~ zTTV^>fJoNi-^b+{C;XHC)4Ruc-Qn9#Eceu-PH^n#eKtWVagmDRUg>EMqq$E=KjHnT zaqrQzhtlmvy7i-8xz3lPw^e>mW7*%oVZDK0#3%1~L;o5tuL`&{qMeYutzWDQl?{of{)y`k|q<-K3&wuB?|F0iE%_LDaQelm-cmsoxiuK|p z)vfnCzn>D|-Fbuc(_WU@3@`uwtyS5(PAo9Zt>tE(-=^7bU$eZ3cR$E(!TRN~<$-6y zj7RQX5}f1fF7mvD_w41VuVQwI-PM}wQnb}Y07f-wQ4GbnH)a4QOS2waf;TB6sMzRp6|ZSFI_h4?5v{5tFJ%aT-G@)NJYz= z=c$)!(Dp6Px3uR@+oGVk_uu1l9>;h>E(zvpE}f*2JX0lbhDG6`o0I0>n?67P5cgE$ znte_!K}%gDmvSsv^J4YtwX1u-YTS%C_h4Ii&8_Lv(jLv-I_=MllG!WyXFvN>Ge>Fu z_iLNfXM1>;aJ_R2Wbrf;6*j=^~y*F@ zRj9RTOtxsi{Mu!9mnWM1S)F|A#H!V%>FgqhOPzh>mn{AIaPhit4grZXC2Jh~%2a|j zU!62}s`T28s}sx`ZM|jUBmO@-9bfnR(CpLGZQuSrR(@!E`<2Nr@|Q}_pYiDSTkG<3 z-*3s5+q=F~Huli`xaj^vOPzbIrn8vDZoKo^)+#8z=1|?y9S1#nkLrHXnrUSE?Sx9G z+NQ>p>~mkTr>g`{T%$e7)5KGeN&3>1$#MTH)5W9}1h)UOqJEyKppW1uC{OrFU$FhIlKg^!}?PKrtTW_CfZ@L-R zX{lv>I?3#&@Ja4{{U;`5zEJDdlhMfe^EvYViG`Q!=DruZ|ByRf<>encSxNVth=+@( zzd3yTYlI|c!1FUxR3~W(Ena_q`OeD|NXPF zH(&R^ypp}uV`|3&0W~|rzmumPdGYye?N*J7Ct@dvF-+ScS>y%3`UwbTF{QG`F+03V#FC_nsf8ily=%7&4$}6$+ zHGI!+`)h3q{M~jm3l8P5iuwZEudEPpCUIEiX z&AUnqEkC~dv93ZH~cwwgDnOi6G;_;UXa@yT9m^|HH5r-p8s zRqV+*(|MMsd4GbWN6?p$%@(Gf*>W9Tcb-j*@-DIC-{W9l6kc=mi=5qzsfrI)@7x*q zqBpt7O?~+i74_#ma+BwZ>Z&K#WGSr;47_+TFmPhn#fcWhS@-%*#?D@LcYjg#_nEf! z>!qjv`Y^>c_V8)z@Z8w9zptm)YrT=3wtL;iH@iOF(bWHHQ&V#5a$&O7^QUL$PnaJ6 z=~Zn19H-dG3+Kug$)7a1H*->xqNdPKKc#z~T|pX*r{+#oGfEEL8KSc{!@;I?M)9|~ zMsrv3MxB|L+`q(M?o+SO`squGpQ~nm&-=1O>;Lsbo*M;{6gySkH>>UHz2T+(<}cxKZ2Gm|+yY$ki`@%NI`7Et7z`(O3>7lRbuBagIfL{{piWo-51Tp!kB zmb}W4KfFxxz0o_9Z&Lh{{Zi*8coVivIeNZ#xl_$ozop>|^;!=kJ(efwN2Q+a0DYhB9e?ep;wWAbaToHZ}!|Lv{6 zJ?<=L4WIY%T6k||;koJW&%Jr$F}wQcG~4xgk^9m<-{a+9n)h4s@8-TN zoqY27wfNoTcg`s7IsSjEh0OahwT@=F<&}N&0-{qp|2f<0Ud%uI_sg&IZ_=bvTa0IA zhUN=u>xdfN%LvNg|0_O~snCAT&8DDtnp3L1Y6?{(<*d!N>4bHx;p<-;)?pX3{zOHQ zFV8BC8ND;5e);GquYXWDU&Y!$b;DbyC7c1AO*f@_nD3ZMhDB_*|7N~)zQE@9v)8ZN z6TDsXWxsWN<&{ET*XELS|6?w(+`pY(WB;S?3HRaO+h;AaE6+JE&8j2sQw&XX1uE$Uj_^6=TlXGWV9cC?5!x`lE- z%gS9j?Lxua1=mj{GBz*jH=fLQFG=c_$oX4L!eJ{tEqCb7GM2h9_u?)_cC(wCk3Hd9 zEEQkzRwj_&&Z_$Vzf0Ng>Kd=~A9N`SN4k$?%(c7biUm z*pkAdGRgB?Sh$H`RqCd|e{28j_%o*_;$OtI3zkBOM<262)k|Hjv{b9oYvrRLy|S-b z?Nb(aF+NL3TF7;O^P%3=Dk0O32utNAcP^i)K38_z-fhihYu8?{pR~GP@Eq5%9OaKw zmux!j^2ukZnz)$SqR!Wsk6cYJ`f}p-6WJFZ8-ptPRDyO^?Jx?{3}#8}yR@c6yJ?M! zw#ej@)#q&X?NQvPRZzF#vNO}8lxfN?MZAj6tJdD>yOyRSd3sIp+K^RdFV@~U9`JtI zmVLJzE**4f-WYQH&)PiK1rHJhziB4DscBs-+E~R=dp@t1Z^zsoF)z<^hEF#Yv-OKj z(#THv{bEXr;hRIP&njojM@5+?#r`E&+o3= zHQRl~;rx5g)Niqy#qa-i!sSv_?9H2jhLexV%{tp(C&|e@r!ZDb$>rHX=AB!b!sPQN zd<|-iS-4)qRKVcNul>b*4?{OkU3qxH+G8rYORfsC%?%W0T=*nPz^bAknspiTGUl!9 z6_->R4Z6WVVEpU=44kNBNvV6u2G6Dak#E3-3FRN2WZDo<)@-X+%+3ud)NnM_@oS(&*sbKRC{ zTS7CXFFi3S6VgpOwCrflS{W-wnW>43wRIma=DaCT9De59jC&G=&2RN?`$ez!^)Y?& z@NJhU`xL!%d#on!D&AeRG9joqxZOqlhEnlr;W<%rm)RuF-sF69HOK8Q)?cc>GA%h5 zFe&ZDMxMIN?Ke-oemH$jTAU>J)x}1umEBukn@(J~c=>;Z-LiYH_p{aUXa8ZIcHyYT zguIG|Z+owoGt6hd=e_zZ_uW}D%w3+a8R@RNII%<2J#W*@)duEKZv7WdY-*FY{-cq! z)!m45neMc?uV>5e{rc;^&H8sf_LWz+?63Lyq}H@F`}T6Vx*z4v;yOOJjUVlc-B-W& zUQt(wN!;dLH}CE{R{mz)Q`5=$0rStyl-)QZqfBo8m3yYU-@SV~e}<1!(4P%0x%+>8 zooD2^XY0zh=g!Xc-M&cvYr@{&g1-V;w!E3>dr?ituj;S8)!~ZXXhFeqiU7O5J9rCIM)lw4a{_yuvtxDQbb~hDl2~~8RnI295~Yu4M6G&$UE1M(^X#81 z6VEO*=Il>UEPEPybi2yi`}6n?4!Y+0dd3&k zU6#AjVz>KEG`%=Wq<6tW(Z=q_?Jg`yk9Qh39rqO3Z=3bqi(U23N+XxV%7`l=r>mF_ zvPcPS{m)=MSvkO<wm|z?x#)CduMy!YWK%Nj*LLzcJ%-znUYnX0`|o?QJSqnm*JcbC5jFTdv8`Lt=JlbQSM z;4{n5D$SU=>v$pi!}iIMy>hdROv+a{-UBm_lI{!$mSgjxSDzBX~GINrK$N#)_+rP zlsdbvH!acQ^DEEUlfK^%*zlk>$bY(hVR7oe*2zh~j{UAN{vGW+wT18L<;N$tT$bDz zuB4(rYlik>)19+~d^fvY*|6JEzvISC`*v~Hwo`I9*sK%uWm+yT;Ees~%8;k|Al=?Y zSF9u2yX9FyQ@PKnyo2uw7V0GB9eKo|xF|eb(0*(&zrno5=J|FE=S_fr;Fg zVpe_eof|8<(oCluJ)@P}ksunvcdy%Ya>u{N!FKmsAJ>09tSD4|+3fyF>30jA>r`g$ zOWw!5ukQZu6cNVzmloG6%)a+{>i+zF7EIssj#%Hk>v#F3g-~eZqTkc|roGtqa>csq z^z~=U@9PqSu7no}V~twDf3LkeKtD ztBHnJjX1?k600KRN_bc7t`Lqm=>O$FYQtUD8J5b&H}FO?hc{|`^LBWk7OJg6I3LoGY~3{bgNt=ldn^XUo^DPv=nnwf#uiB;ED)^Tq!hZMoX_ zFyQ$XO|{vYS4&Q)$npF)nA4WMMRZn|SkHyH|K_`!9jgw`Jh1qRg~k00hf5c;niowh zZF{h=T18tTR_Ef!KY|*rPLJF#D+{ab+Wz+b{UVj!aS>C$-Rb?iU-a_)ii<)!W?tad zm@ND6$(s0eT(1z_l)B5 z)9c0KBMlOxe82sQpI-UQ%6N9M+LQYmb51+!O!rBSniM0ka`oKHQ#qnpe0$X*s%CG~ zbh#*WA;I)y{fY}=EKHx=)R*sBY$<~mq+`%<}fV$Un$RBn!}p0$ILl$g4hP5*>X#cI@$L_D;VXH78S7?E!=D0XME@lhy(w zeGl#lK2gnEEIQnss){FEa}c>HxJ0x=U7saW_egN=glnDglan2z_$6mu*u|#7duf+r z^+jXLRORKo&A$#y>Z%vqI`U-7WlPz{Ev|CsGhZwG^VoNvLDqDIecOKP-pdG&jw_km`_H3G;>DIy-vu4h>cUzm0 zn-hO;V@=@ls2zLu=2liar@#J`y7-Ex_p}vrHaq8*m+!b8zH7GI*_?9yH(5D_<*P)E zSKU1u>n(Llf1j;^#qGQI8iQBNT)b%$o0WE&gQ`w@Sp*mFpZb{i-nFe8X0M)QOP#8H`zjU+HZv71maw7sN1O9aJHVf-B!HCS=%JJ?*)r!&_cmB#d9q+ zfBGC(F}X`mOt&mi(0LToS~Jz`U(SJfVN0eR)R39k@>W75taonVq=TZJctu{p-f< z))lLpgY(uMbhY38O-XNU*IG82$g@ACe|F{Vi?fzb42$o0uj}~2WnG~CYU`=LPlVR+ zPAx8DdHT+wc1v7R{5GS@w}bwLao>M<{^Xgc(>vcN>Md)`Iq^XG?v3lGuBTgNJh0}9 z*gf;}x<`NCdH+#=a8xUPueOK&qupKh_a)v9%_S#AEhf+Y?R`S+?j7!se|x&tpSkCI>sT49#56Kn<`aOPA)yTcc)>KbVZAK(M{!MkZgMDNn?n)PG1m-S^~7QGAisM-73;M}0QN=m;KvY%IT1dPl$M$IUrHMO!}a z-MsyuxOVr=qnOLa!ON?m>n!md-e_cLf^v3aY~bmv!v+F-UTc4wA=h-@ll$0(UA#Xo zu$TT~?AzkUx>YCT=Y#$0mz2h=a1b)yZ5;L1t@QI9jZDQ~2Fm#to%E*~SoCgw$RWPR z>*yqrmM3Md51%+Mk9>0Etx=?m)YbkqI*g0_#pkYxO!%&GjQifk7k4>Kf?^ZDhi>QF zzWJKH#ez)&n^!S4?k%o*`z6|3!NqY~eoMFa?l*Iv=|A3+(PHwTW!1yQ2RzGVe3!2~ zaq7leu2YkvMV(du9pw&+t@?L&=7hOB-&puPKDF3{U&Ei{YKt7lT#l=ZsY|pUeK_@M z`_BBMOV~HrPcl>1ohUhxbz)`R=I_@VwNAw6X*Skyr34C8%CAe_73PrtOaJ%9oWD;t z8?Kw0Shui!Uv3^v;T|(HQ!uksfZJ|vpa5Rd4i|tkV2*_gz!^qx+dzxqVLX^WVihb56)a2;R#=!S zM8m@#tOFV>a05YtP|w9GSQ;uAf|j5gfDXnr0&TVc2XicFxw%5LnSqI(fq}6l$_jIE zLc~a~h?ELRxsVm+#*lMi!6_QD!W^Dx-OyK;8<^@DnH!rRuP!$<)Uz}+F@!BIw=mbU zurx5^LRwyKZfL4!VPs&;rH^}+xtW=Xo~fyUg@UfJk%^wUfw_W_ff00nfT4l0xt^hc zr3vynbFizSK7_3^H!(2>t$fE@9GV!Kn{go@X8UGuWliwaZ{~Zx75F`SqH^|%S?;7| zo(ofRP6w^kuE`|rEYEAK!5`}psK!{67~@@KE#`t8>9C5#*l5(}6lOlDhDJ~<+P z-TxQcf|b$^@#V)ad{n-#w3v%g?l$9uEGhdh?>cw&_x3XioMFHJR=AB{wQ_GZ;~{gV z8{z$O_0R8C&z~)H?9Vlcc+09UCl+q|crARx8&(%JH}^NXJ9bDMz9F|oru4V%_NlU7 z*BHM1=3H>M@wQr@RDJlh)B71sWOx-^+x&N5-d^%(`DKP%3H3#rA3kpPli9lSw-Uqg z=?tFSjy-ehK3%Te?5``Ir1{5p_A1tomG5q^$#L9%C*bS9rM#t!rEkz#?cKW~_SrvTStMY{FmaN{ z>pqFD7ut^^%4VJvMD7HeW8ukr z=gKUYs?l6J?bP}MMn3gYpWBmfZT9ds*vECad1KsWmx$$&3@4kEQhf3|-k$M3W8S@& zRrlwV8`mwS2jzTDJCgHevG}EP9qS_0I5XLg$b6DJqx$~nyu&q141|BrSH2`TSLlmd zPP*7L@x;A8J?D=UaP>L6WjX#3p&n2^b+HLtr$z-X$W3na3 z<-mPbiNB2n&L_=I52=)3{JY`5n#boKnW1fI%FmMHpTGL4QqlZTa>u&b4eO&88b<8b zU&Ou6oAptdq1>DOD+3F;S0;wv`GJ)TcBLc()RK5 zg-+8JwIE}k!#@*la{Lwil~L{}w=UJcM}L2tp6~@3mE@(d7U#E~IN$t%=TnK0y@&A+ zMy9w7Wv0weL|6;ad+0oA|qC1S<*PH+S&+=u$7O~uCw{F-kZQdwx?s?!s z;aT#Y#x3jTB~^(oVw$-2tm5e{k3F_7v)3?sm)pT{$Yr7uv#Byw|?T~ z!s{oNe~j9>XWFW2?pxohY>yV&#Msv-%D46kn+4slOiX`uI^KKzig?d>M@zA?3V9x{ z7o~1Y=8q-zd%m!{l^FBBapR0Tmft4rvE0^nGb#H?3{QRMge3ikd*1KcvwYf-MV)`Q zB>s()@@IGcYp3<};ExvD=(V?I$3G}6m6u)6=dYgM@G^*Zo%Jn;(mmS3Rvmwi6!t%U zdg8NsqTM;=nP(kqkCYrqI4E4I%Wx+tUZ7ejx~WF2`XbYelYNW!E4+Hv7} zFS9A{_Yg_6Jt0$p&&FhSFkNcXvWm+?~@NE%hUuan`dD_CE|7qAN z8;--?t71BKJ-(pt6qnJ~%$B;1eVX9gwL!024o-V;JZq|WR_zzpmKx5ugxQ;zf35g; zq<@3cD}Ls{_OMK?gsFGrOV2T8Z|D`@5La&~wr2XGch|iih+SE~Gh_C%MQ0g9|FIv< zx)C+!{N0S}f@_#6w}{;dV*OMw$LhPEO|uGk;e+}#2i5b9NtH4> z2ez;Z-ST`>F1O6K<=1VV((j3D-uvFjJ+M#ySV`b^j~_;d+y#Cu;&h%Av+x(=fCvCqF0 zK7;XT*qep7fAB~gW_Qrb<~w3-fZ)1x1|K7>9#x-m{J{jbFT zw3=5ouDc`rr#$p$oEMpNT;QNZ@3&>#3A#zDwoI~-or%Cv=fi-^eOd9FIn zH4*d6RbArir`M@23E5$_k5!B1L~F#{16!X?Jg>WP|Fl1BPt=)G3pX~{$g*WFd-y9g zvC3*&rLWBIlTV`)-uyS2_n$Mx-XyqE!u`i7%WLb|gPzwnB!>sgU(mVt;s1+2{`oh& zww4WC_UU_iflA5=A3jI6ON<`u8&(!Af>Y49-r++`) zu|akDdd<(%{r)VK{(Us=^Y`3;|2bD!R|~#Zy!Uti-RT>Q4!LO@@{@74U&8VCMDvUJ zcbJSBUtO4X#mse2N7)MbOZo=9*jI$pVc;aYXnX=JM#Mao+gRzhxLkb2R?**+;`f=bNavVYulERa-02m z9LJU`mL>dg@44TnaqvUZ622w90-hYw*W$Gl{!Fa5ZvVI6@BhC)ALj>siEMjpvG>tT z@k_tg+dBU_zNOCgKYP{N$i;Pk|EpIWHEzz%n8RO@_KWTBz7tc^=l_4#6YO7TyEwe* zd;isi{+S|mTRx<={$KN}pL5l``yuvXzgah24}NIP{Xg`_xkL4?ySQG)LyyAuT`ngKg%&UUV*88nZt_5{$eny|+CRB% z6DM~VCTAQw!N9A*STE3IlEI_pSbahE3yX1cZ`03%haS8=*!95q;Ld|T4<2t?-SoOC z^x)frvIlJs@;Co(Zg0Nd%H{kppgo{6qvFN33pz{o7>F0#vnCr>Osulzi2XIF9U-|audQOIIplE^~2QDb8dIf`7LC5{h85ox3**R6t$k8sEl)-P<+#L?Zo1Tv-(Wu zX0u(n6CE76dBd`8->&T{JF%1N;LaP{HD-7xmF=3o@8GOseHZ%!lyx*F_D|YffsAnf zuz58PUawR?HgCuCC*Sg==f+edo$XwDe7aoyyB{_UkM+BDsmB~wmvui~v*+f=4`XUS$)=sb>iW%%2$=*#oP_HwqxU%p(-cUimmz1-c(FJ~9u zIDN6aZ(l*l;fs$J%l2)vpH*q{c*XyWs#EKxcx%RM=Pz3Rec3bDv&OTQoBC(YyP|Vl z=W@-0$n}v+BO8BT=P$D6^k1H{<3jAEPp`gS#!MJye~Vet)lt#!!!T4E%~?q%*(RS zDF;7)5IB>x{ZP;kzO=(mGuY1`IK4qCw#6&uwNC}lxx@Pt7;PAf6U}9si$Cb>VcLI4 zzkqRGbLj`WJ>0JoC3xD88Zdo3yywek%F5hhD`MRz%HXQHAU{G z&$bC`pO^;u`W*4uHu3EdCy^voGYii6$tp?e>nF<|ah#K4SJ}MBvuslA6WvX-cXUXb zIkc!u7Wrg$(>bWKeaXaFY0_UK`O}mmB$p5L$y2_3#S;JTIrIyYGKr>#Z{MPNUfMSYsJi{6;5lf zxJ_O$*JL5zD-~Z)<5!FFX88nXXnnn4GRtr875C@C_GcB&dg}2`u$krAY~r#xgZb8k zw97J9)9rLLCrzD~Y<$DVtov*t|7O-4!Rj8}#M3v8cNFhC5SNyHvuayoZc6kGvu|wG z-Fpw)CHxKuw@+lhnfOig_pG^T_8T_8k@+qDSNw15KQH|Y_g}dFvi{rr&r|=RRMhD& zXAbo2*ex*UO4dK7`ic9GT>mirr*@5Q{PF9L*KKNa|KS$bYW+EFUu*u;^$+bf_vRn3 ztzxfl-hZG2|Lp!ZynibHssD%FKg0Xy`9JvoSpLWOpQZf? z{*U?v)2+_*e^~rS;(wR?bM+sd|4je4)t~nNc=?a@KU@35^$-3(vHvvxN8~?a`{U<7 z*ME%vvHlO!|K1vJzmMI2%Kx$dpB(?lbc5ik4AqnEx*J6A9lLx{?FLWv1NjePXB?xi zO#YgSyl2w+$De=R{ZqKF zLH|j2fqR|H-O2MG`7ieWtW#xn`bW_}hWLZwpF(#8+n;Y|Bsn%+7!NBsRcLJ zI-hDkwBez$+dZ?aFEchi&Xcb{-qJZ)&3|X{t1Y|V?R$Rj_q+NB2VaUB|1mDARer?a6Q&HRU{;rl* zl3s0=dvDX%t*>%#$KU^WY_@gz-mSmi+P>fO``&N)H-A4Yu9bgp&oIB{*Ujhb{IwsS zK2&z^uPyy^r1bvBUGFRImhX9I9rs;!{qMf(wZ~uYdt7?|qy1;u!*6f5ci*}jaATXV zOA}P6y98wB3L1>;HMlGL_x2e(km=(?WI9CSP1|J=1j4 z5lPhxD=aSTyd%yz)xAQa;BjxStBrw(a9iBO0J)EeLC;s8cBy%qWqALp>fwk_+p@jd z9;+Un?k#jQoTIf@`ezsOz7zZW7~@SPuk2Y1VF|+YXbnRdgHWv!FUc|Oy^^UV{ zTv5V&T9brcgvfaEl?W+!Y)JTcL2zQ+UhgjJrN90z?Ui=*IR3CPQIMmgBckYv(uKgc zYD(oj8CUXZl8@vh%*#(+96R;%YEg->dC4+@5{4xOo-Mq(qO}pl8}9oFUf;X?PQB~K z!sz3ZEW@8I+jJ`FyTtsCu&{F9z6GS>fDX5f1Wt9uD14u z^xV)WA;rxbyyws8oDo`6r_6j*X=%+G{YBg*96jq~qWioj2T3;CegCmKDCp4bBz485 zH^OhOS9#DcvXsrMz3fxY7U$IY)n=;gwI{=$o8)Z?+;~iK_r)AfmY!wW%*y)DS1NAZ z!BP6Qc;;r89*qKFE|(+bciNRTIUmf4>b`yAT7cyqQ?bMTng^pM3hq!!R@&9~bgGy1 zM7>w;OB8-}Ox?dQSD$~*-yU4; zm-3dFCvR4rFZ?2IDf-oN`s(TI0h7#EFSm1dk>2&^{346VUGCdE1)kSWwl$h}e{24N z?DvkU`yQS8#FD@J+ioqf^ZB)w`hs05r8At3&^+dMK$a$v2}t z<9c1QmiE{1rwdmcwcexY81QIeeCPe^hD?70jhlb0x7xOPt3zg%N{`NK4Xy=$j!qW3 zxSDI3+M%UC1TWp(5twdLGIi=)=6&a;KAq~kI$88Elbve7&fa&+c5ADPO~20VSDN#C z&lQj7KbM|ZeQ}BJ<8O=`48-R+9}(c$Q<2~GqEn(Nq9vkn!qp?*T1}OFn68{ z?}i6^IW{tF6y7Lb$ftODa>d*OuPto*mhNj>)}eGn;0V_dzay3{ZFQ~Xt#ZP@mfUmQ z!QuGQaF6yE4ZqhK_jJE-wDnz3w%BrhRg~zHykKTaUtz|41Hpo&YD{M7d8U82ygJ`@ zFYDJ^wfMgB)Zb}eiaQiE%D!2e{j>Zbuvg$|M<<7*bGtzahn7IMpthi3OHV0Z;#}v5 zbp1!lM-(&KIFzdtw|sFBEerPj9^PtIaohUk@h`RoioxG6eh>X6SR1}n{!)3!FUCtH zOR8V)4t*Jyk$<4(fE8B|Yi977=`W&Z8|B`MsMfyQ`R3!Jy_$pmrMd6K_Xtn3jsvyJGkQNWu2_Y$y zxa2cM*Dd?EYWAm)pf!_x-?qjn?p+wWj`jEp_r70&YyPhIX8m!WykKy0qRx}k9pZj| z;wH(#Wq~&LmiWGF`DT7;`^HDNcktvN=43g}S|xq-MQ+ZM+?)eV-$Bu@cf`_#M=|*& zkIWCHuXzWngIxrZIFhU$$vB0z7xOtLIwf^TUuiV;<=M5o;Tyle>Or7F`Q_ zKYw2EJ$aj}BF*VG>#O&F2rUkJYTKciy}io*yUm-w3o^Hg1(&s~;+*u>ZtDv(|Lpg_ z_GVp_obfSb>UT%SZB_CgN@S*(-Sa^%Kcwy#5ExwpVrA zY_H;eyfmuADthnP0uIZM31)oT7P`MRnXu06#{N)KN!h$5k*hQ>ez~&c%9AXq%s*1= z)qE$f+L)ai<(hp>YHQWnFzxJ})3O}1*Z0lkdEUW(pe$0*r6#?BA^b3xX!*xA5jRX1 zEiEv1nG_LzYimCHCDTpW7A3OJx?|o*uTZPH$<;IE>X`*)?PdE%O&b* zNpD8g-JneqC$uh|A+@l0HRrp!t14=f(w1_Cmr4sH&E2XHdVy`T*pz8aw=LJ6h)u~( z*L`v4Wq#h%&gygV^4~)rZaHyjU9Gp$`Xy7{id+?@h21j!G~+|+_J==z))X$!t(f8# z6t$^$tMY!4?}9Ja#4f*dY2DS7hA6(8C-D({s?XxHEOV`XxLAJ3y!`m-yq@XYNwqrD zH94kTeyOlx8uzP`EK5I^oxQ&EJ7 zH(#$2NL6}!MM;id%=7ySR}Zd=D@t;c7KurI2%osDQ;9!b@6(1`!pikiKOQ+f!|&R3 z!?{O{EdReuPidbcvuS1S%MS-1$ppx0$aQM;pP6lTOLnEiv(LFQakDHGT28g|%Im(k zFmd9g11J0@K5X>c*PF}!-}hU>#Jgq}GL?Ds-i2=}ET~9XzG~aUZATjy&h;`ceL401 zt5y5t_T34uTfaYZ&GZbN_%BzNUcH~PUFc@4OZ-|eiaiv2WuL>;k8!HfTQ4jWx$8GI zZxNgAsqD&C`?oX)&%EI$dT(3Z_vV$icm7D4&$zchdEN^jj)_-y)IR>E#9iu@GV^Gn z{=}zs8`j#FT1QSk5oB;v>IH*kx!+tP(K|m|*Z=roOX9+F8CM~k?Lh@tyjN+ZfZ_Ce1(8|4$M&LquGTpO9t#FN3lH-Sk<;cilJqQRU8 z#tTJDnynL*R(4E1B6du!cWs+^f^@Zg7(1IswoyCKz35C)WA)V33?f z^TiJ;IqJF|;;*K&9p82QTchyr9n#UUe4!ge*Jo8Q_X#X6vD;=VS$lo^GbMepSXSF?A zli&Jr1H-jL?~k}eO}_U;%xKmf|7R22PZVwHmr+R1sJ_+TxA^&u471L?2dy7mo$mk8 zc-%&ObV{?O>fn&vnipZ5~G)&Qx?R-}m=q zZH29e*o5a-(l5;3IGy>4^e5(b4*MrcbxCVa&0JM+sw=#~&#nDa;V+ZB{nr_P3ICnE zGlK1OqHctucB*heNb!xNJ=->{ujIOaKKaD+n9P-1oT8N1PA%*Ho;j=j!L)yYZ$Fpb zEIqzhV6Jqm)Y`7RzFJog7Rz^CkBE7{RHIP(fpL<{y`>X!F7=f-pW{(X zKB-b_soRi};S`ERridKtMS{B*I`zZ{G9H;4%{_UZqf7^ zZIUxuEhEfcA3AlzqfC8wyKbuBH^JXswTJaKMeRw7IltGYxxIq#{h{^`EB{3Pv;7|@ zrBkahciPua6_wj!bpLbDol!O`Zq~h7MrW1IhMj$WtLUSM=(Ypd;rm`GsNduXJG1S8 z)2!o@KOXsHcRr$9 zOXA16_J@ed-#m9TzE1qp_J=z68;gHSMDL3>o5cRo@+L>9%irCZpZKzm72jwtXql@Z zwU4W>$hL6e<}a$(xR*ZQz2~RJXkRTAmpb_b^M+P2<`>ypJ_q(yPYA!D_eFW<6Mix0 z_1oFK0=2*Co@puk;%(7gWg|XMLHUP91V`nQx*u|JE^9M1?kc9A=KI7bdssVD=qAsU z{W|~V%}fzeyZKNt=-9j|=Pw>M5dBtcE0g}Z;LGoqtJhfD>wi^T9LOx!aMM83jQQO} zuOG`(L=21=n*OLnZT}sq_g#5zUs$Tatu1R;+`WB~Z%g`J!=t^EF7rNJWFOU`{xPbp ztZ6YzucNGdN7$R>8#>Y`O9G#-pYM zQep?CjI^udeU40ioOtDJUQqh3_*XL7KH4u6Z%x-?Lz1!U{F@iwEESm;S+d-1r+;P4nst42Dj&6aF7hIJ@rPwfzbYE#1E{WZYuN zFcx^tP+~V_LFNm63BjxN2Roi8UzTI|EXcqe^pwv=DRJhK?~F3d0`klj+K&tFb5FSS zfXimefsGb6UNV{UAMFfd`(VoSVLEfl0yma0W|p@Mif{C{==HI!XEJ%iJRz2G!ef=Yst~E|y9d+LW za~H?xtkHh6Fh=7|TsViTL|E3xYkOwif3=1g@n=a%he(NPG+O3Fd@B6zXq|Rva;dXI`xBSy_X_m_pU=C$%Ho?6rdRv7 zcZK<@-OK8enQu8PwA$-*V41M;h4l$8JqMVyZ>$yDeBI~l>zw-g^+614enu+p`7}pw zx%0wOV-G*SW4w{y&)qOw@%vv!e$k|wy~(#PopPLi?TYMA%U|9h-Fx>7+4`j&vb*5i zuUnP!J!$+DoRq@Kk)Ed6?tGTGQOBLjp4xE+?w$o%K4rD%GNkJ5}6IGo4#B zW%~~8^r^cxAIZ|rxTmw;b>3mU>`hVIa<-PnuW#JRs(W)^?c`^zyVUvjvi{EXU-dG= zN-lgyphcz9+!Gn)mt=SIF!)YXOKME9D{Qo0-d?oI=O^E+GW*+mBz+EfZu?+;>vG$Y zs?%Z{%Wi)(V|PEVAW;@VFm`Hs`sEGJ1>+rZ`pM60F7Lfj zw6N&m8lm`R%b>|~HnC5!P0}qYl2PfOFkRf;d(zVRdN+^WSI9e@miBpq*5$Q_?erSg zTnyYfVY8D`u}o#K29uUb(p0m9w>@tZ32xK3h-&yh%iQ}Ar@;}kIV{?Dx3B-EI)|sR z^8;`G>7(B#v$gh$mIyANBeZPdi`OL+el&5*hN!llEW5<55);e5X6p4t8seuWEc)u{ zGHaEj1&8&Yb!*tO0={paaL44SPTd_%uRX=$-*{46wg~*MxP42k=gP4g?Ims{FVuou zxXfMi4@(|<6_D+{^sR-xu_r%&{wkATw@mlfr>5@A+4!mP%ep<@mg^<91Zhe-RBJBg z6?_{0c>S{YvsQUe)bi5U_WC`#lVOyd+Iy@cac$0(2Tngt+^p3Cj@8`BI4-oS#Pia! z(?08#2_Lq)AXU%NZO?MkkH?`yF~|5E(arS>FPn-=t*Sgz^rGA1^kl&w=X;I)J;Tn;V=WI%JauzXaI4dq^IfjTOy^xb zqT8`c?^;un<*9kL&upuoc@%aDT`*00aocp#z3PCIX?clJXTRO36xm@P^?3OdpLro- z4j(xT?7cs24w$wk(d4~w)TgVrb}bK+Rr?=x&+_nO_1yheX1?htThx5wQf|5KjP|0< zB1_+~KUpt&NkKZTrMmE*rGY|0bVAoV#kn?WsjEx^&5s%c&&r&zd5+bybEo{um%S|U z-1gOBqlc)8?d|I|V+VM5t$-hmC$zF@Ml*kJjg-^R&wqcd#D%QO>{=RA5yoyzH zgQ$sa$SW@HSy9{thg?M2gV<6l!|K;v*4z?v=d6jtpPg%?w&hHe=VHBZ>)Ey}_s0*H znrwM?SaW;ttgRUdN25P4?w=@~wq@VW?C141(zag5?n}M0SmC>0_4DG_OW!WN?RM{1 zQrxw^u0ZM2vp3s?JGsPTGglbz zT=sAUbNRB+1J0S<4?Qz=s}FAJ%{AMbZpEt{j07`xLVTtb%7=4n+c0i=FOq{r>`XGOsTcS6G^oCvY zidy%yXp{9(-_+)NE6iWcd{c01uBV^ca--?)n6@t(_DMkL=l@$PM;sW^S^U1x%P{4 z=5FP+w>P}_XRGhJSfBrF?EOuZANlp`zE_v+n$Yw-K>Wz|Xr2P^8;ag9qt?1H_jGr@ zDt*azA?wJjN|ES&%j9?8T>gFL3cI66N;@N^Z{HJ&*LbM4!n^cg$hoad{l8nUzW<>9 zLFW9L+G$U($ffXm-oEbMea+SP^-+&&4?NNydX!u+owK$tWnx>591q`p8oq`x z?_%$leK6fS-^p42{f;-c=3U>gKFj~IN38zE?Ac#Sw4L&lwpSfb zZ$7d1mP%6c%$X0i@5~U`P_DH5(%Ccf^$r?wkMY#jA#@2BHYiIdL-{?^saO7+kM*1zwQpu(B9E67WXf0@&u_X zlf)cfhekar$($7sBK$DsX0D9r(y3ER7e5wv5G=6>M6ZIyWK z-R?hYp9}mt(fs`O#YLq{-aUDudHclLcsZqDv;I|edu|@-dbd~pm1X(Sc`xKeoudU; zN1VL6!dh~bPwzc8wS}KN`)bZzD*ZL}7jxCB=^CaPfs!>+y)j{lN@`gh`YQxX*{8Yg z{4}%U{FT!od9yMe<$B+>Or1QX+j3RW)L0Si)w{S@Q+-2ul`Sj7VlrG9-mfmd?7h4E z`pHY*c8R}S;63-{vR&_P{P2^x#Umrye0)1cvvaEi`U##^QT?r#x0Ickq7pc3RV>hKJZoK+53QH4*SVxhmRb+akuzSz{)Hs zD?YuIs}9WFx~AmPlS8{U{VF}QQ-ss;?B;iObj!45R^M2-sc?#Y9qVCHGBEWD7&*3 zw}0%L__D`bGJn&CHk*vB6urdlA2OHiK3nV)#XNNj>#}Lj=55h_y@WG+_iU>fUr)a% z+7_}oWOv)%B@=V6B$=5{UJ*NK#kCA0)5%*Tu1Nik_%*@6N+{~%X#uv$Cv(CaRxf_K zjZ1oK?(*ydF0;Zt?QDci_+R&0?bXkjlc&)3wuGU|hV9J4t1s$aeBgPw*(anU$d`T3 zv&pBHa&>xMEjg>UdO}CaLzC&Ey9!^eYHxfv<@Chh)ED_8qFmv>KDEZpcvXI;(&4N1 zgazi`rZ<0NlSnpQ&@1c2|7OCg_>x`;!}W~MI1T?LFOJj?x~i`?D`|fISMeFrKe>Ib zO5}tloXcIxw%z2|m1C?Qr|P=$v#L#9d2F^ttZuZRy_W{3z-yyPo_U$ieDWOI1YUl9 z(ZG9AD6{&ke*c{tuQkJ}R~rA{u#D9{^0Dx(FD6-&!?$+-zms9iaWlJ7YmsQ&9JlU& zbEaiUTz+M-)T(!y@q8WcR6FL6l2wwYc!haI*-kESZC!;isBnN#Sq4g@k8mJ?Gjk9n%)_Z&UW_pRvZpJMXN# zGF@!jH=kE8xt34yP~Fw&RsVXz632Cl?-(_igP-r57OZMH*D!jQ<8zLm1-Cs{dMIa3 zoA4lX({r!%B|a6(t8B&hNmnVp=u**qaQPhJ*8WgXRvHBSJP?P`^veBxn26L5B7^} z*GyENw7$eceQK=8;q~|Dgg&16!6EkL3yZ-0_q!gRu4B#pk@5YuyzXDiUydbuJM)ik zPQ5$JUZghSR^GPWJ68X`X2jmLa(m`4Z*w{E(8Os+#Jubu&$3v+xUp4URwznv(aCpV zd|4j~*57#cCa&7y+%&1(MNYX(ox*1an6I4udRf5CEA|0GvlQ+ge>e5n=AV~@DkiOM z+xkX9ret2*<#m_eiS$olz8MtVIo-2=*Yq~Nwu%cgWlxohwd57uf1>$KjD1%dTZZ?V zzp~9X&6n6J*nhF5^-Rnw?%}>G9`zzp!s?Zsm{-|?y0E2U3)E`l_9<>{FiK}$mMKtV z{gI>7y+fq4i^F5)%a>wqTW7wMiuKZXnKtXu>2)WT-TWu8YO3PShgTn6uzfdc)xON6 z8FaGiQFkat!MP(M50mB@92x- zeL0=;A}(xslNl7?`la7>e@D{87sn%WCl{86EtEaDn9(bc?WNeZ8$4-A*)JuZ?De(2 zcRKg##(cAzL7yhhyFX>()O^v8Th`R5I_h$E{fy7^;y(I$t4(6n&7fS%G`)lx|peV7m9xewVpvTBieslKs&;6j;L3`*wQvmHa&D^oThrobUD4MDIq|%;c?G1+TUH1oiHb?2Yg$ zY7{*?hc}V0nrjM=^`Y(^IorL~TkKodJWnLSSL2nR!&T*l9*qCBqIN!CT>ZW7+Fr;1 zc^kQR|GsayVdMVS*Hf;pS!z44!tR#Ot$UyUR?oAWB6jADp{Z4;Nqt6LcJs14t<%}1 z{?jjRJ}PHm@y+(X;NL500=)LNb9kQz-=em{I%1C2_`-#zqaS7q`Y1pb>-P~ z*{8wjQ&;;}c*Wk-Ji_{M1Iu+E8wrKZ=W3G!IwwUfy}aknk)7(s+9FvK?{7+#+hkw( zt@uW)-@Bl2XN{+xFgj_o~mVV6U(I^V2@v`?l`u_`Wq-8NPhe zL+)7bem3*_qWya6@fzhi5;qdPmOlz)ynbO$*!liBAFg#6%AYkjA8qk?+Puoj$5LDF z8s)zFr&<25p*W*0taA6So@MDbR4$gJ@UO6sU9fstu-VT?{a$gCYg^1xTM{d{J9qDx zvHI>Vo4kpwFSJURpV}9EMZMnjD}Q8k$7e3zu4NxjWvohx+W+s;dMB>1hx63`WLf)g zy>^}^6u0QF!|Az6$>ooZ%{zZP%lh-|t(SH5z0!50lmoI&Yi-|5Ed3^+``NPd{LRyz zr{@aK74>^&vNByPFuc9=YbxJE!3VRROzAxAeRE0o$6UX=3IX08^GrAH-Fl$*<~tGg z9Buvkwn|Z-e;&W+EnHq6`TxMewU@r#>Wz2Jo*(bwW4~eE)==y1Jh#`{eHY#@w`Qi= z8vdE`@2&_qU2EGmVK+lm6nBp9U2{kA*S_2I-U&%$=x*dK(LTN9w9b^m675vpc_Lrf z76+yqMeDd#Pdcksmw2Xk+n$2SEP3-6xXli|@IisS%BtZwcS-r1mWJp{S(jz=b{~GY zO|o=XNjq<|+3Jf)A2q{vFN(;uT72#81f4L}ovM~SLdDu*-G0lj25qs+n|(HmHMi^Z z)J0paPrUblS^1u9hk?hE?_u*RGA7Q6@qL!-6R~K@lXs;rTCXitxxTYnr2oyAyQ$S7 z?_0T&RpymlWv*TOF8tS$aN#Zb#b-{q}kgf0f=#`=+%kE%kG) z^6o1(`q!e{cc1m$anG|PVTaORg;u&0d?kHaj;fIWIAL>XPlMKXd-o z%-%6`b+Wp-qN(Z%J;$w&-t*o+&LF;Y#*VwL?~E=8bWKVzobBTh=o1zlG>P%Kn=f0M&n4-OH5XLo z+G(C-x0+HUyI5`V`4ey4lrO(7=?R>bVkFtSAorC5-xQ_!hB-@)Wf?s&onN=HZu-2J zt3AI?Q!HU!TJdNl-|MGm6PHfk6rac?#`U^(LQeUcZ_^hSJpHmmA!YTMTC1NqH*Xs8 ztaub!HnryL#C1Zd0!(262^%?HYF>G_bJCHe>Hc^ADu3O*aMxSOmiO~w9F9zT?x(w< zGrfEzt7qY%)ye7GmWgE;2XPmamW5UxS(zSweX2Y}f%#9}X3mx^HLRxw}^?baG5@&G)^2Gj4k9yK&@b3H$(IZT@n7qxA*>a=*CxgHPLo^>|5eeay70;@cV_%d>Kv`GC7)ng5Pyw~sdnHN>@znm6k zZPvGc>U~JL{J^Z|Gu*{w>||sYzuf!e{9$jG+mROyQ#Ri!tGjV{sZ8%0gIUZ+)Vzfj zt}hb1a=_|t@S~##TlZ&9{}KJbaz*R>oIXFxni)DBeUVD(x{N}s0)x62; zd-G<+rSdIaxsz*y%Vw_Bsa})mQ&zdz>++j6f2;Xkl{cSM82+5f7CkvWe}j5ueY3x8 zyy=CqBBAxCU!TjIGB^GJx76Ju`8g&I(X)zYm-(DCc`Wmo{g}OP{j-SZr?WZ3XQe&c zUG%r?`-C=OsY`A1mOWGYtn`_y_>9h4gM5Se#>Sg(9*(*CW$K2@dyl;Pn6bdZd;aUh zwl?Rf3kANUhh2_7)8-TRFE2m3u58tdoV%JIQ}~M2kIJ7E{qwcXP3Bhf zq56Qm-C?h;D%pGg5TCWCETq(Y4qyA-ms~9~))+qC>6z0W(IMJ*!>sIZpI&fYj^Vo# z8k53p%758r3VxSmmz($7OW*bRsT&Uqd2bsh7QFjmxOhJI(z9>wSH4{%XIypT?6N6m z7X0fwW3XkYdevUZc{cv+=Uv_>t-qtT@KnBP=4-Dy`eri^8_Z8HA#r$3MIpp($9z6(IvX3v&vj3*R$^NVq*X4ZY7jv-~ZIbv^ zQu%74hy8nM-=Igi_R=IPx?0vW0)mHrNTp?R=+b=wk zyEgydQ!6i55x(c@Yfh5`q{5PumuAX{T z(`~J0r2VwM#N?$bk998Ny4-zCch}lqx^9avTipJkoBFIraT@2emttv(P7(Q8%k182 zw|8!k2s`9y(SLj1(IkiK7oHhr+>y)s5YeG}dCtVH#V1UjR9sYd?0=}%uUx#pt-{n* zYE|aES^M3$YEC})D7WZQdfK*yO^>(LmVIj7Jo)u}xoGZLS}$a#&s|mC*6Gcm8Sy#y z5bM4-jNEA(5A|j~kFyo}Tlsz*8Mv|0X#r5FBCMRG1+$+>{^{-CXlwZ&Ge3%`lZXZ z^?%#<_1(n&ZwlvkEizlSIK^^8`tifcdma>C__xMK>+)Cgo|MN+ICo8CTg${1m@)0` zhLV$YhOU?ITvOY*bNSx2)n3*9`L`cDv75JW+eb}j+w#-p)_Rw^^H}E`Nh;b|W!bv= z+#I>nNvTVnL|d0veUTDqTCVIpDPTuTV(I&{d0~y?;mkuuRh`#o zE>FoiH1p=q@C`Z(yDjg&_WK&3_Vh{2oGFW|ewQu}aXkDsHz@H@Y~>-x874d9Zam`O z{zhtl6^p{Z30dMd*WQ-?yXS5D3(F+?vI#Rh-%8eAUGwns>%I&3j$LDx+xI`!=)l)M zX^&i^6I6H?9ba6pwU76<`()M4)_rw>Zzg%}unm2{d7X9kp_gT{8&%AG^QOu8&71LO z%E!#hr`2-1RyJx*+M_*Fz5AxA$H!;OGkwowe0xWu5G)zaXX zM&SIaysUdx@wLBJey{i}o*aLE_vG>kZy&UGyYRn=iq$!N^5ABPce^X3Y@^lql3J7g z+jigE`K#~Px`@=dZr2`7Tl6AlWx`b5l`OH7-bfmI@Nr#B+35Uy>xq2TbBk8AJMqd^ zzTgX48!>T;QE}AvgDck-|1P<;f4;3{{iPj$&N1hne_=hHe|CJe^r?XMjQnr@Qx=w- zX#4!IsM&Foxe?3TyF5qBBBxKAbM1t{CUY(kiKtuoW+mJBFHDhelh%7#Up!+ri&W}X z?k%TZP20NT;wiUT)Ba84?UpJvEBPjF{^t4n>jgb&JL)SAe{vJ;?Ms>a@xP44^wd(z zRCm+6J9ajzdoGSLINHu0nYdpoL0pDGF_ypOwzpirYy6ythOg%K?Mawt_o7U`K=7z& z+0@-yF?ULrx3VZ6p1t$(ru#*Qgfl0qI&pX>R z>aCAb!bcCO-H!P@uMSn}eonU9EP6h7&vglpoqnH(4vm?#2o6>I1@bZZgF%dZT zp67&S_x@)=^5qt57+0SXy_q<7^Us5^m*d{`lay;O;DZrI@LFT>FaYjw0wd+supIrV#-emvj z`Yc(qy>UrB!I9TqltlHs*m7%u+qP^IqsKNHFW*jDa$UK6$20~0Bkvbv^Riti{ut~K z)F$R>@gVs7k10np=J7f%aQxECQ2TY>4z-hRm#_cR+pao=b+6Lbz#B_`r6+bK%jPAn zWBPclt9$)vkBSY($qtJ%xVN_Y>{X7x>ONzaTPCYjZdBAl9lxwh)=d?+-Ugp^tN)Xj zy7HGM z&uIHj(ONR&$kG3Qog-r;UAK91L?P(in}k) z^2$+j-|H!8y?e%7=h~K6Lfd!52AXfs);+W8j`(rE>O;RD+)IzYDI#$v@6qGg-fktc zFSfN$`yyeqCgpeA+S14`GjF$ViQKg79cNVO_q%)N)hu~eaOhfH`<1Mu zBE!58zoL7#Z=QI`u-O5lvkNM z!|vCI#s2qx7U-ARs#~@Fo~Up3rR7_Dyn$BlROu<}xb#1{)?9g=ox!u@sa4Tst0lf) zbiZZZ-f?Tmy*Ilow}w=A7jDQdcop&1;;}^Pp6r-cdiT@6t-lk$v}Rh}sXDE{OZ1-= zvsLs>dYp0aw#M9_XHO|S-SY7GmK8}GW?kGZ#vXOElRe5nJzegCu&eyF>94iF#MG;M zo?2YFP2s?Oxy-#4OH}L2m-X)F)!FgMV|K{;6ulTODisY zxbo>s*O&88o437xBH#1BM4LCW?PLAcZ#owWMclugSyz^xW^Zi0d)k&sIsK`Bb52gu zIX&$<+n!_gEy>(EHxC|TbK9+)`R4D5-J7FW>cS@riTR_ zii$f`&K>pS`!s1n`%VYFCm!<-a|?Ke$6o$DF;!tFcT!q*)vnd6R6Xj&Lzs8Yyf4}`rTNF|Ki(W?&!qTGT>L5Y zQ{7F?=*Hrdw9U_bx>?^kd^l5S{M7kd5#zKIddU~>Z`raY_hD@G{F#5|fAL@c``o?X z>d$P;O1~{Ue!Pg^XV#t#MaI|9?R-D?&kg78`*$Dv_*MSjhuimUzkYsyDdUma3H3OJ zmnHpk2-WRjyeqH!W^wH5r zx7u>Qy8X5LSoZN)TYFP{cJ;Z3!mG1Zt+q1X_IJ;!Mbqw|`s^jH9{OIStjl`o+<9~F z&E0qRP+sBoEw?w{-hI1#$IF7vKHtwuiRonDP|W>)WcDN7Yc;DA>krSpG<{F`mOEF! z?L5c6)O|0%iT#bD6)&xQr*3|EXswm{Cg+U2FJ9-Q)@Dw-eA#Yyn;L&<@!{2nY9DGJ z?r*P`DLeUa=fjd8=84bmd{}k_{rcG=>({qzi?eskYAY~_xGQUSF8*K3?nK2aHeuqHJ10a(Ty~8% zbv#xbdc9MxF7*08y?x#fE!H(=@D}~e?>cz(k<+Bz<-)d+^KSO|&40EtJ?iG9Q*j=p z(FbRz7d%Mw6YqZ(dNbH@OXbBmKPU8IjzYmWmZ~p3jf> ze)atKojz^TXJK^So}KnLT^f zZN=}A*Nl_(h5xg~SO)6MSvJY-wDdb<9fod5f6HY*n#m~Hy~?YcE@ zR_y(FbN9BH`hQOM$3Hr^{`O?UYxC^)@7lQfd)ry*W6{sH#g)7*_&RNFbk(!#XZgH% zy~Ez$-uYNtzP2YS>hP8MKU+n=J3QO_-&5(La=x$03_HPC-UI&(QW)drAF=dUs zueqF6k;;u*M$h-H*ASQZ{(0kVi}IvT!P|ZB>|D9F?c392gRHoN_50t>RQKeKm2Tvi zc)e-rmxe>p7g@7b>^41N%5l(3UwqA%0}cERMU8Dy#``>dk)^~N#qVa>$~G-RBPI%o6de}Xw)_e0GS>blOtrdrt{+++Cwco+%=tQYm&zp2q-gPB=EB3DC zcTusLRomFX@i26|L;jzSEAPuU`S;a2_5SbaDt=ZvS!~C_%dZ0k7v%qpkjdS&kaJ}I7GQ!T#nM9=4@ z2{+!{4c#Vw)vK#|h2x2$eTFlCY`@%FygK^-hbJkgZkiN6E~rZA*?;t0L2OHi40G>_ zKL=MRweM&>$0}4`_~6rlw~X!$x0YL!$88TXzkM!mas2$-7v7ds9$m0HjAyl|-=mUB zm;U$wQK6ESJEcx*)6H1&d*8J>t=>Odr*yqQ*stqWW;+(W^F8EtUaHdhLDJWv%I;mq zWyRaB7G5{a^RBq`bxKXh-|pA88;=TSDC+J@tM9(ErmSV!WI0p)nwTB83#Wfid?)x% z@VxUqPj-i$L%&^4Z`Bm8cUouIWf%Nt?=Q$_zb|(vQ*qDBGwWinKKS;qK8s=Zs`Gn; z&+8~PeBYnE_}<&_bBo1WUvB+);Lg9(e7{ZBAFD5ZC3@)K)A?4FGfGa)s;YSZ;(e@NWqkgHFNZruK^Cusg5HFDwa@0;EPcg_CZ^<+|IWKf)fe#f3m z5qI|0A3Xk3>U8;sSgv;uylb_VfBUuW^EK~H?}XNOdp+>zbB&$8=-O4<~(B+beAUNbjEA z&hD!9m6NAi{p0wx`eB}(%Jo8{r;iTq^6<`>{b$FMj$exu0@g0w9KFzddw7U;?bWQ! z%WvL(6a6gqS@f&@>JKslqnpG-^TM~YZ$EDoYMgOSs=W8-G3M(n>t>ctelOg3`^xR6 z?QQR-$$hK|uC)3P^5XlY_fGX6eKp>&4(qI4B_4v;!T zijZfoPVS28+!vI3R=79oo?rg_>xG@=T<5c5DpokRo%^(!?|Rnn1C>`=rgHu|YIZ@( zHY0BB981|SKj%l`!E5()3angq(AV*()6ZS9R}HOOH-2o8cS|wtIkCL6{8j0rnN@SO zrTZ7IjF*4g>G9Q8(B9{edzvfLn=ZO+Uc>g9*`8P=VzzF+<4 z(A&Qu=XDMI{2S-}y`vjAziLi>;DgY0-fI_muea~&Vki(lbFlTs^F`v!8yfNjt*3B^ zPpsPc^-tBp@HfF_Ma+L1HaPEb@hmX3W79mbb&hI`cHN}io;yNo+N}>?Z);V{YrLT4 z*0P>!|ARYMs`@oI@2dRZpk%w?HqSzf{Z3OV94ARx%BgY8TYNv`NoiHfA=Q_5kFvjd z|0(pdD?``+pJ6uVyuVl92%M?Ry+f_>neny=7A=*E zs&^8i{8K0X-8yIQ?VYhJf*Fsk^r=2>7iS+PE!cYM*5{Wc|6N3Ne`SZP>iM9!BTLI$ zx#DEdIn(*`xSrfOJ?+fwtB34o?9JJ9UU2@8u&NsK@Au0sSgyIo+CeqT(f^`TOa|xu55V?mESI@wdy*=~Fmc-m7dD|M&RYBc}Qj zx%G*a_qRXW7I$x3&dS3-ug|af`dC{(cEiF?URyS*pICeDXJ2_Izg$6Ji^DH}{^$?Y zLCcS+srSpxo+D%Zw8Feti&@~f{ONxt&j0tG{n~f-$iJ!OPHH#bUk$7gDrao2e6;?D zspcQ4Kcb@Q5^r=?2Rp9P{&yll{NwIt1~+`?m2WkSQ1E?M%OiGDy@uZ?qR6g&isl=| z=6_{u)h8$XH|F`alW+IEqz&`meAyrSNAl|Hr@B0C3qt=1H&;Fi3pc*uXmv%8&HD7K z5P|i#PA7<8xi0xet@#y4Gpi7f7LVA;U8mUCy!Yt+Z#gKpX~(6M4dO!c^$l)3zp=1x z{=vFSai06R&soYIn7#dzZHqPo==`|d8I_^Bhw^mt-69-zuqRG@d7>>>_U4AQ%F^3P zQx2coWVNj?=f<^(ytl13B}TiO<-A6b2TKaJF5UF6YJV{sm>N5VfA814?j4y>65{>y^;^3mZUzO6C0MECwZsgt9|QA}&Y1-6k?pjHO$0<`JQaWC!u+5jM6j{e;)bv5EV4CI*oiK0Z><606276xzr*ZSE?}Q0#x*647hOW0JaMdnW zUMg{TsdMLDqfUuTez%6-DoG{}dt0}!?P`#&*M;?1#Tak`~UNvfSiT!lr#a zN_nROd3K&sGX2$dt^e`zv-Nx{Gd&j0(H8y~{lTvFeB}|_Lo1}$<*A<|Q4^M2@VWoU%;Mb!wCIW_>+!NxCr;M=*1FGIX`fIZ z&xMbzc| zecQ@pN@CB~`z~u$H=NVxGrlbqeB;tP&z+^ST~A+n@b{{ZvFDSvTg%pN*O5NisU+`N zqZPTnblU!t^RDifSZvrKQKa$mhskY|6LA{)9%k+P%K~0J6u7mm&8)!mlv#IK`a-j| z?x^K^Za<#ew{+X1j8jJ1+CPFciz5|7fzcp?J2-RjZZvQ4(S9;WBZD755&o}?AyPI;7zi{oF z_UqKsg6j(R?zdQ-cu=w7_{AopnAr!`dHdgYus*+`aK6i%>MpB)`+wC$>mNQNEOlo8 z)IU>tOSR9iOYLO2oxOzfWo}co`~^9u)4Co#x6;Ix+8)}lb@JJOe1^3CuLWxxNNWEIh%U3tiT+f}CM|o#dFr37Z`FOC z&E9;boJUz~@r&w}Iopl1u5|Yp&bN)2^EGGIHFl3>3DG|niJROwVb+zb-duMosV=$W z=HH!9LM7gYB)LgWo3wp~r~bJ&-M2m|R3*%Q?YL~F$l)42Ueo0D%e+s0`(wN|N$qJ9 z|I&Lw$2;8ES8=Mn)%hIy-hH=hu6^jU^T*R)97?$8^Yg^UbWT}IDc6&iDo-Ez`0b(9 z^%=U$PSs>I>dF2soU$d-EIRd+(zc~voDUeS~uPGztt{# z#$_ls_co6h=dHaPAD@!QGA*6PXOw<&cgxIg`n~h)&;5BNaV()SNbYFuj&IyQ-WXDf_1;8YNVzxd(-v0SEw=4> zX@z04TtAf5+%nE&yFX(^W8IGF&v>&!O*StoJY;{Yoc)?V(^|eA(JvHNtg|Y3w^L~C zkv{cX-KPI{ihPn_zERp+&o%YX?!L`$Hg;4e)LfCcl-%}hg_%K8#_UTgx3Ie2npKi! zGWVjc@!U&p!W+wS*XF&H4_eu*botliP8r>kS$j1zwp$3jjA7Znt}M4yX<@EmyZf3o z+m4=)nrW!KPi^j|9u?O~lGis`nKTs@dvr@j7F%3hw)WhNDyMEo-McN*PnI0J;8x^m zo0_JpYqlu;%M4cDP17V#XI<-|19n<74`7AJ+W%$VL%!#ziGuJFz{%988kK&Dj zXH!<3nUeN2sBN10TEUDwhfFKK=wOS5%)A-vnT=bGwDXtfTyD5!$WS!%V;QgOBspUv ztCI7lBz8@03E$$V!ybDkDCy=GfeoEHJWqu<8777+%=cjyG!{Lds{1DEnv}o9(@6_X zoea`)Q_Ebo!A)&1`-N8r>UG##f=)!{9OY3+>b||?(50T$3ynRM4c9r|-?ZQf%h8D@!WwbiH+`pF z>~`#SbUvNhyF=GEIls?nc6Z>Z?9!w6B zzqZ~mxZ&1@q}**67F6Cj)2H^O(5q>ipXARsE9|(|I8L?LkTEl|Yety-*(H-Rk6eq} za8Zy! z8h@=Mo3hD)dl!~1cy+Drt|v$iz)M%}*X3gw zZEgPEZxipCzBc>5{&HCUjZJxXmzBg3s3O(?DBg*k54*v4Gibd>*@ zjstJbvP6+-r7J9>6S#WzYjhqaNf>UO+t(^JFzJotoz z-kiME)@#<&`qJixPo6@7>ViQ{wYS zNAKP(cprIb@mHN_sYt1hp32iqcP(11JpcT5b?&bIrIVi~AG&ZY?TgJ+RvyO$E`wZ8 z{zDvhdF-@#`S$hlUFwlEE?paceB;Dc$?VGOzaPAl*f^=PZ^_h4lil2ns+aoxb+Oj{ zV=eupmt~5S{Jj>tQUlART#mL&GuSROsI)ffEoqp;ulD)whd*p>Os(JlO6Xjg(7~o# zcwAhDUG}`iyw=yA$AkNhpDK7MAt5razI#V+=9Zq!8v^r-`-(RfFD`EWT(iS{!L~ll zq&cn1%6Ayw*|8rK=+`@xVqmk_g5ziW?$T0mELTgIDi|n0534>y|3(wDH9j<r=D*SV3m)|4u3!{ECk&m^3%~>tzd__@71R zzT3E4DV`MaEI1|G!RN8sc9MvTyGlKa7x$8u63v%Fo+VCN*XF5+?{-L<&f?{5R9(Jx zrmbDN)&^GnlXs{6nZWG$R#kH1NgD?XAxFnqSzDJX@IHTjKw+z&`4iKb-iJ1-RTj-T zelVf%yVtuz_I%|JH#(G-o_o)y&!eZiX0`E$9l>QA97XThH%?l27E9!qAtDFgy&p#C z_kM&Fr6!doXQY<+2J1TI=cI7y59kFTp!+01cXk*X80s0BS(?Tw7@H|Xn;9GH85SW185-(Y7@C2uIWbT$wJ_E*u{1MOFf%vQGqx}@k5w==RERdUFw?U%Gyz=~VFtU< z1EDik!OTJ-+SJ%c&)m?&Lc!F+P|wuFSi#iLP|w`Z0%V7&Ipk&s0|iq9Lp@7VV?zZ~ zOCvpVb7K<)BTEZCW6%vDAXge$n&?>?8CohBD41Fr=ouRuDVUm>>RB3sO|gL9L1JQL zu4iOoW~g9dY@lamW@4ydY-y%v0=k|Aq|3s{z*xaR!Q8@F&&<*gGYg@uuY zf~AFlo{5=-1&SX*?uPo&Cp9moYSG6OSovVveqX0*=XBM5O{^3K@HK6_f-51zax7$k=u;p-kBHw8ZAD9?z&`w=+K9 zRuq1kHN!{9x94`sMV)<-YTbsH{?1*s?}5>*@3s4$|Nd-y{>#^rxzVMoUR}AlI@Ez- z!vQXVM2A}E)W6G%kL59S{E}vP7aL%=|I_h(#$gORclZ))?e_0k8fUY5%jF23vqe|0 zg-*F$e)AjCks3aQyAihafv2yCf7|Q)_$GteigoLB`0MX35q#^t8azO(?g(K6!_KyOKUCg_oW`xSrV}{L(CAt!LA( zYrmX-g?UD1Z6&Y4Jf=evI$Gg6MDa}`b?<~s*?OJz2A-w&?W{raDYZ|96#yas`{+}?_1*)KL!-x_~G;-bg?#x-si zcD)yvqq0Lx?Na4+4)*5dk={0cd_S@K3(NCv`R0+SY~p2j#f~v}>F>lPAB^XQHLc(Ts95>U;REdXY z(&XMZ?QW9yPBm!$AQ{BQGc{Qu^p^j(iE@9U)-+dhW<8O-;BVhvDdYg{xkO$S6c8V!x@KjwgiXy*v;ZPd?oV1vYpBqH7pnR9?t2ycVDq$-umtR z*FH$?`tGdP*nTWey~5FoIq%Wgv*+?}JpEkqOnS@3l9%jczBJoqE%9 z%zv_c$z!YJKl*KAdFds39~CWpHcTkl8nsh$na~*(iGxqhrJPQE{xp5x{NI{4b+^u+ zX#X|o-;tbI=QeE0_D`!ee7^O}zmxhI=fqil?~pn1ZdcSN|DPQBy9<9BNWaOs`SYZG zg!U=7YTKOTva>mxR;O-CPFJ3vT9>?~^!djp59(ajK51^eBYz^!`nH~B=}GRQ&hSmi zcN~8376~0O+Iz~J^}1&1!P2DstlvT@0%thZ33oN^+{7o;Z)x0H;hY_KCt7(wk8rEv z2Z3(WO>9#Ark6RkABtJ`y3@tm(ES!mc+o$U-q>(JUb=tWE1V3+ctAcdfT3Pzv7e^ zSWY=9G)2**zEf~_SH`*co``vlMvF|RD$Q_uziZEgjFKH;OT+H^{7!RPb|}MM?T4Jv z)##tIl9$?ix_;IqUG3H{d_NNU+outHhEvv)T>+a)Sk|}d?nn)K%i*qyF1%v z{3%+xy*d5ny({z8=Po%Lps)sICRG=FR>nw1n`ecQTX+GmGqk2iBuTn(3Qy&V4U z>~_6x;)_lHY${(~U$%A1?;AUxw4ad6(C@O>52~r}zomQS-NJ9SS^g93%{T8@?6cIS zvh6vi_0PY@ZQ_H@Ju(yhbk@G~XM*w3E6ce5epK2oll&t&WVwCZn|AT-I`==#DL0z( zx#j$WM!(1na}GYTJ1@P(UH$FR+~;=!V)*ABGrbVc;ZXll;%VoX2a<&>KYu#R$d=+> zJ;l($P;}cW{Y(F*7EBf7zRa;sWryD{fp>y-O|vBg_D#K2CH6XUVoHcFUl@O2>XlxH z#$wlLCo=W^Tv(uSvTN1MGh*2@bYr%=yI3t+G}mzB-AC(Q+*7_cb;65xGp}e{E-jzG zZ{^=#%XuYc%rn#|E(rTJVPQDm?yMgi=T8J$?!33>Pt(O@uH${u_KA;{EIJvy%uicy zrrqOLGxtRaV(Bk%gd;UG-?YH@Lwa_B-@Lf*^U?kP-Y%UUU-@U`N&QFmb+UG*c24!|SFxS@<@=KI$@~7q{fPS;%fS}e z$|s%i-uO`7t=|f7YmML5w*Q#XGRb$A%GpWpUTs`-ynjpilG87XKJL97>%8k*-JAR0 z_T9R1+pqd~*<0K8Z})QE@ZLYYKiNRy50Ao&29|~f*#*h&$G7#Z_4zC7Z53rT%UZ1b zN$s~2=`)Y>EVd{+z0ve|y>;2?kMe!>^=qE)kbh8d+wk|ty5q6Ot7Y~+d-3n_tBua? z*JOS^JfYQ~EX^q<8(FdBi^r{rjPvjPe|dL)&G(m0{@ypNK55_i5mmuDUyiAdUv2%~ z7s@tTHkvvAx8L&>|6bC2N4UwmXN^GKvlC(!Jb7)=1p;dtCx1}i!5nsQ{zJ_QnR5=( zSp~vt+TK58-yvJqHZ`M7`Gf45_Ol=Ke@uMm-haq=gZQ@tn+>G39_Y!V7a2{{)4<99Ci)-3#Ilo=06PE!IqUEccwimfG&vmaVTu)Y1jq0|2QrC2fZoVMT(N^(c~FE)koI7@eFJXEjX zus?cOV{y~N6^ms)^YN{-7z-mi)m` z=IBEUCR+<3_e8#&Lor{Hf=2p`5)YG8q8Y-H^{~{#1<&;;aht_k*DGJ!~G9=E12%Jhd(qmIrzRn?7(Ep zr3GhRH2ty3&pWqb;yl5_>!r$`UQoOl*=&6I*LQERMDN`#<;~` z+uC@aPt(qS^q&**=xndI$Bc8o724N74KMnm6@NPPa7A0_y0t6UZ@YVK-QhQz_Fc1l zXZh6F`%jXkZ`y-7N%6uD!II z+py_JgqTT~gzYw&&osGpqv6>LGpx9++K;Gwig_iRA$jlU7Kz874!eYR?!IJu*VpXH z%!othj-{Oa8f|ei)*{o5^ZK@jlXGY7E+~?IU7Fs$>%djBTl^`0az%6AbT}@&XVYbq zn_HO|9v2-MU0GdlG-tba`Sx60ot)L*r)|vj_Go!~P(1wT&(C?Y-(5ews!${PzHOD? z?CPSE8!Ym8oX$-VYVOWCto8Ggh?!e%>ElzL(Y8B^KJ`qRYc(}Od-;@{#V%8W&9Z)7 zn`aWX>9T9!jPG2rd*7TrdT)V%rdZD6KS$dhl~pl>eF*F<|M0!5jOE_)(r=0(Teuh6 z&Ha9Hx%^w9IYx=b%W~y5{=9qk(N$LF$LRt-Y8py2TnuIyJD&tXTd&asGni4+0N7^({BEz4ImJvbOJgGuhmrSl%6b7`?j#;@?Nc z@Bg>%tMs+~yW$Gh%5A7RnsqU_zdW}8>-Due@(!2&d)*hSeqHzYht!X!V@|(|QEQEt zu75lCde^+87LGF%{3fWK2#WNs?cVRobH4Yy@9Dk9-yhz2UL@+_xll>#(7MV^hurrF z_vXzuzxc}6pgMMzdECcu6ZYI?eJ}dp?Xsg87iT$lUnuF1-JBVAc1`B2MV_-3G=|++ z$JWjD_tufs-rJP!8Qzr-l<9vt?yL84{Y$&O7v}qMl-!r% zUEDLB@$T~Pu3UkOHxwkD4D0<>oYr7P8+4O4a$`Ivy?ZFvyr>_w6YTe$F{6aW?O5DP( zlTPo;gL%KsHE^)J8NDU3sVh1M;eUW*x8_s*UF ze&qOf8SkIH4 zpDwbl=l{N%e(Ub7=W6D*b(8PU`7-~x{rUQN|L@CxGiF?KAo3AcrdY=UCw>1de>UuJ ztcqY&PYD!X!LPiRX-R3xoI_iJAt$X z;_M?L*`1*m))-zB$?gsn@>$fBCiyAE(0@^l*45Kl)dl~{V#B#OEqC6{dV1pDTQP5r zk4I+uv|QExY1FhvcZx4-lY+NaYEoo_pt9Qy<+derFa6Bqo1e7O&tBB<$*L_9bFJPw zw_W%kaQ{%y{QL(ELGsgn3At1mFi*S{{F&+5&c?+~j?D(^k~McTf8~>z@O|#d35Trg z7FT5?eQ5bI<&T}q!mFF@p6@r~GeP&e5$q1j`@D*)*tMIP*z< zO>1w-^691>jE!sEUzJ4UKFVm%+;aAbfM=PCV(t8W{Vk8Zzuz>jvAbgbX|B?Zx1Tqj zYh-@G?N$3+PDtTLz}u&OB8y}MC0|DRsUBo4lNGwORQkE-{%wz6_s3Uk&pdg!vGz;O zLDd_t*Vw#wC``C-exGIE-tT{xe~A0CP&__i|HXBT6V@&BI>I7s)7<(;hd1=>O4aPh zsEyawY|WD1YI}?EZA;q4loL~nLscUM9SoBjRWt(@Xw5qH<@}bpWnB{bU*}pat$z^x zSg&g4g%IzOzB5sFa2&@&R2eBTKT+&329FYF6GGFc{bBFEb*4eWWlZP zRvfcR-kkI~d8t+g8>8d))~8lqQ*6ESuU>y8{Z;mt+~0;eC-c{<%%?onhInwA1?O?)tj_;d|N7|M}A&YJBfld2OR}IeQuFyCc;Fr}#gu_+I7D zCG+d)>jNh@zOcF!XkVuFPg4Ez=FXo6lfUHrA68hM9AaY93$1ItfA9G&K2;95PR&K}c z?E=?7w%*D6`C&ezZ-w%O@Z5iwpR?UhU-^%xJul#?WF7a^f~F3$lT4KlFWzI~-WGL1 z>E+I#bz0U>Q!KsDO;4Th_)Emml+E1}7YjbUaZznS@yv=f34)D&e;jwcGk1 zI4^yySeagt@TgaQY1tJob|%wPXQH!~PgD$;s=L>{Y@O3KbVN#c^q;DEMzvF#)07XT4{{dl|1#;zzxRKae)pfFAlb`P zr|s%tc|o@7V4VNzR|Qrq8AXpTvBb=3@%7wsr7Njhlsm;|;-;lco=39Qx(c%_@7tBs zDED|#&=r^TMH4Lcg*vM4+@U`0%K;(rCIR8=i%%qR>0kOHVe5MS8uu5Wy;1oW!(YAr z;#;-*-?X{_`<3m@au2nPW*({%>r8YNEEKyaw6Lv3R%md*tcJ?$7iOSO#0IG#5iHGg5~MQ7k4c!5ngm`_Vw47ENW$n8@lvJ8o@bGvsd;AGXI-CVKC67j)N?Tkj&@65Eb+*H z65?h2-Sw2xQhmjvHy?>8oa{-OW_e$S!PVvr*Hh{4%ac+yx2;(^Wl>N5*7aNSx4hr* zexv*bZ3oNaPMoUUix%$8;CZU)rW279aU_`gu2{EB)Woxo0-QBlSN}H9{1~Eny-bt4 zWcThMj_VeeP5<+$3b!V)oK{^Dvw?q6g8Dw|^-u0u&pR-2^}3ms|2OWs-1yi+gV#h{3O>BZ$e;lg4E$Vudc~p7C_Xz(8(d;#=5;kA*%FNm%J9UxWtANrTtuDW< z5o?2HhkBYGwFr`N+Pe6tqQ#6=EDlrVxIPS;cv2{@qd#WZmF?%Z2mGJVc_Dn)hQ>-B z>&ur<&DqcWk@>;XF7u0P z_uRjs8%lrwQm&~!DtkG8X8T8<3C_&%@-O0l@p>BD8U+6d+QrCrz~IKq$Vo|OmtAUn zZND-ZI_PA)b z5j*d>MHx=#o}VdNbup07Xv>onBWo_D=Ew!p?{rS=Tj#JoW$pf#6K~y?d-FMEzC*&T z39{+8-)qhP>+|`;;@OXXeGRL<_oJHa;hzJ`-~X8uXk1$r8G3Qctph8~qik+2?VCHZ zIC+-+&7zuu!}Gh-*WM`f2>WFfT*bm?yDNCtt}PquzAgC`_33`tDUL{yQYD4!lUG>^ z9COW>R(Q+e>Jt;y*_(WhE(^QNd3n7Wug}D^?vRvjgV|@yu3nOiKX*D-XOqee%gd|o zau{3;EGxXR$@1{}er}di=I?El?nGalboqNuKwV|Taeh_z@8wsF<}Y;kwDJ$%RhBY| zSDsgL47Z8Q&e#>U<@TD#e782f-4^E&^{_g*%43$# zE~WJm8wze*S$UXa&-O?2pOjm86{y9|+57(c_ZMGoH{IvzEUDsM_c5i~HKkrfWcJw~ z@9Q$Sp01gC$trX55)Xqs*-5t*5~@E$_xHw%t=Qt`8LD%lW%2vRix%#A`TD_5)fDb8 zvNKbcKY8eP=1y7K#|yX3CTrBm@7{gADW5lYN${7B1{r_*b-gv4Vsh^=3deaVwppF& z@-@AZ=<%evRa{f~aHfXx?q!y*t-jBE`}g^~%BLF)-d}s&{@uRX&1?4MPe=FlSt-{& z6=YvyxBHxN^PHs$U6Zy6FFd;I(ynt_U1!V0Cc7?V*%09vSiH=F^>>B((JeJgyAIvu za}8Ycc(a0h%eh1K{4x!~Ater5)}9cFRGFo-DC^3?xjdUcJhHae`K?s>VZ&w?R!=Mc zfRY&#CCWdkZ(;ST{T}>)eM*z>cD7l!TJr24FF0_}!GNRjdtrimLhbLgFIzWP*q9mw zOF2DHj4bU^FV0lan|fij?^TV84KuiZa&Z}ME8K9;(9TC`nMAgY;zR2wBKTKvGsA|wyRc6S*JHWy?rF*p2+Q0 zIcK+p-R8S3H~ZwW8OJmq-wF*0T|R3~+RC-BIKPI6h90hLU@_o!4f72=8>U?I_V#gZ zM)~4vcSUxeOt#p3#%583jZod0nGtuYp81^JtX*_;&y}ydvagqJx!FD0`tu)$M{C&U ze0vizcSYun9H)p^D)|=;?sqyd-VOM)cw@t-j_IGGG(*eRJ%F?8avi>=wti&)zt7x^VRw z_A}~d_Ro6xPCmbHpITLBN%WSIHlbIJ+(C*WZ>K1^gqFWL6ji4ww0Y$c9`9w7nm)~n zC}F+o79MfPsy8ourT_h>pLMsmTeKU5>OJZDI=5J702(>5j&=-q5hqx zH9t((#d49V@~evF*YDT)pZs*dT;675VX5OqTXWyTd8hgI+q`FcKkwz$@ZR$|HRV^E zTh~7i{Vo@}ec!)*%kTQEV%GfkM3@Cx){?(B7p{~#8E)9QtWdXL z*1EF~mYmrxrsC0grcXq{#wX2mU6NLllHnmGt%Y8-68qiXtNAX!*x2^+@Qb$_-k)t` zR=PV=a%SWovs0VA!$kV0oY#piW2}w%NPY zX-%KI&PL~UYM?gX$(q8NgsU^>2EXsLY&QN?`N1sF{rH)3uQsmZb@$Jg@pIR$PPzOp z?8lYjtzK?!=24-GxQ+aUZ*CL2?R8u2w&QKj+nKW^S4|ajO|b9gSm854S9s0!=9;}r zRxL^~OBU|%71A;HJpJ-gVph?lhgqR2?`&r+nfX)S_L#rjJVqX2Hkad@@~c^XZ)?+R z)zmS(P@1wZEw1C0Q|G!Gmlc{tFSab4ZMxd@XMD*7F0py9{`_sXbxyi`!*=T4@|G7? zhdoyMmnrWuR1`8cFmkSX^XhL^!qJUDi*Xdj=Yoi?=bKGTgU7EbM5pGU+Iw^s#2~N>U45VfuF#sV5a@H?VQ5Q z(VA-CTttuRe$KP6DZiOl=yF{2(DOg`|QiA)Q&By?M69xqsp3NJ~QVS1r_yJWBsU2*#EtPC}N;U0`vzO+#M)TfbV!ooryxpm>arR7^E6i+u zMFH`TcC`LdzOtavd%~)^gN53UgI{MqI5J!8(b;4FzITM>JP`VFaowXG=dKyPDBafQ zD1Y+F%DV{bG~Fxvb;C0nf>_VoPCE5NdC>}`=0X4bgPub%j+id zAOE%ZYs_};MYwT{wg0(Wl||RiJ&w2TuYXLYoDLIpLN21`MNq!XW!7TO5t9v z`PUC9I99=*(teeyMxVkNoh%ION)h{duhLP zVNqGIYw1&^>pM=a3oPAQeD18R`H^4CL(NPsX{~Ggd-?Kiw`0EVjvq_lGI3Bn)23E* zJnN9y#O9vnlV5D--COnJg_6d9=3TBAS`s%j#Ju@rQvC95|F=_*^Skx@71xQx3*8rq zU-*9Q1m@BJe(#QLZ5C^ug=m^AHs8ve$YI<3bC1-kY=v8HO*`jwoy}RW^Udx#RaqR- z41Y6IRIXflY?Xi3?O&K@uIkqApPaw`ACy$P6y+1=Ya$g@ep0n^#}%%{8C$;bt6uRm z>@B{W@#es}+wzxQ{`@{OCvN|`%6gSv@3~r2k3Iiz*|<{Y!SZs!qe8#txz%!aY?`#s z`g){;zh(TYVLrhs~U0^dA|`NcPAQytw;^{rc3H>mM(kK6yNK&c@(ty=_OF)DuL$wwzrWDY~aU zvvS|U)NR!s%*8*lp1+SfXPMEo^vji^Us?O=nU`yt>-OZ+^+WSo{kM-95lv=$0kI}aYhuoda%=UXv{@lAcbkhIn zpML}#tki#=zFKun`Ss43X0vz1hm_B?f3kY|W~FEBit#HS1f^!1m94$*8G7)!dC~rr zPvi>hN?%o$1^G@nYtXc$z^{3_)$5pjNy~fZe!l4)dGQVRmjjCuPV{B({&l+7G<%nh zk;->A=epiI(W`nke$wkTpSS&8M)G&IcW=9Q@TwiU!L3rYGjyizl@C+(*59paHk%l> z?%B*`a&1mhwrBsm^v{mV>HfR($#Y@f6WmT0=VtDkb?@>&@tK>S)t{J?+}U(8-*9d0 z;o4uHw*0Jl?HKqj+r-iRwn_Z&!fpHy)E=7!BzN58zQ3>V*j*=PuJ~tjPG0o9vc-Sx#d+@U z^Crnw*kvr1Idl1{=Y#Ye8HXoL(boOz_G4Odex%5myGGR#(Z)V&R$27hIhB6Pd$@b? ztaBd@EHYnyAhY=4?)A&K@5HYC+bG!c+;DaFx&)57aaF%>^&e2)@-|xZ)h+$s4}J#M z=!sQy%?XgQ)SQv4fAH%wrZruMS5=mBtxik0vcF1b@>`n|@9!jDnsmIa9P~$_f=Bk4J(lPjvdZ@CMbnXn8Mcd(n`%J>;kE+197eyG*?^eScsM;x=vQ7e|*2X(=3 zQo^tQ`opw4f9Y;e(4$2HA`T!+1_Kh)Q@Qj55{p3V`#|d>gShmAQj7CTi;`1|6>Mzu z1B&vKgHub26{7V6T-+4&LsBbB6!blF6Vp?jptLiT_KfAycdaOK4=zb8Nd@V3H&8GH z#d;8zogHNDAxPBCKmoi!60{uAP{9baY!bvYQZNQDMYXd7tM*Mz$xL+0uTY2vDKIlM zQZTnPisjOG&d)1J%_}KZ0IiDz8=RV#0!m*QDpmm7Inq=B6enW#`n| zx2rVv=H2^WEc%`|CG}6Pv!;r#U)^?@7pzmv0z4MJ{OkPn#5~Dpsm#5JhtoE0+ooIJ zv!>rY=*Qzl_VwZ~)|)+&|Nr>@zfbl5o9%Rm%=Qp|Z|IxyavxR<_{(m-Er$y-J z^u=-S>-QR5{L`rR>-he^Pv8IdTJp4i|KIidTdEbC*|@g;{kXmU_v!jw>t_7_EMNco z{`>uHtN+D>%UPeD%Hq4D&Z)a{`N_~rTjpzJ{?p1x3MhX1Un%pGzt>xt<=nGGGS^)! znzmV8_~epw?!pCu ze@bSpZmT;J%5ru`-J_|^yIU9kpAxg#Jbce1f9}kR*IkXyIvaYr;JD`&JI*KZbi6*L|uK$$Dw}cS_u*-D||^ z+;Yy!F25LVb?RV|$aTwatB!d-T6N4(=eUJU=LEeAKmIIncrEnjgs#t*t^HTJzE54| z627(fv(^9RB`@twyUu&pSat6E=J@^Q+|?Ii9{-sldcO6UmvYtdd0Bs6hOd41FT|?5 ztgcCwYgSlYkcIt0CA+6r^cQii>+rJu|KwTS(k+YT%Q=7ArXK%L{mN(VDMDKDTWp)$ zHce(e>uOW&eZIuh$y?pt_tJ%L7h3L3pA-CRrOC|lC6?}099_H5TADA~c_;T=YsI^} z!gi9|pQ{9FuQv(ab#7{R{}jH@%RTy3&G%lkE~s0QwJxy!+y}S@DxLnUgLQ^11rZTEE?*GIi-__4Ixzyx5D*e5BiKPA4SI;LaUpLKISF}@U z-=){`O%wD$3s(-%-+|7=7Y09ZO|N3%|S>K$yUKCt+jdI~~%Ae+b zF@yW4MNr#SxvMc*bNsU3Ula`}y>Kfz@7^uxbrHGuHc8D?*PMGP{ccbFzuAc&ww(Wd zNhSUFis)?BR!vpAL+vgzrNn*Y?GEL|PIss(QZFp-QGPp-Avj0ovB$|{&n|8;-Su_p zhDwvS9JA+#<$d9Vf9sk{^UZR3XQyxKUzeGE!)9`e&))6HdYRWReD$?0 zEuX_@y@>7m9=10f^X_hzSn_|e*^76IQ(kJ%P1h35oAdj{yIaXBM@{0-%C?`|!ddqu z(q+b;we0!)iN9ldRo{9mud7kn7|POX_xn>NkYnSY{;`)v0wQ?~^>X$JRvSt!@iB6Z77%9b~OozHdkDFyE~Tdem9itAha zM(T#$Mz?(ao8J~rzcD$<^5U+bkI$7l1){z`Z<3SI|8Do($tq9dUF4HrH?p5aKKXVd z`&-Q0jo07o{FS?YW_6D+cGU0Mezx_?j9@!W(fuYzh$d3xo5du%#xhV2RB_jv4Abs>eAx=!hJonBxMSZE%Y{TR@Uzg+V=92 zZ*q#X`?2YAhtnUsOX&nm3bwtzjz|8Ih-&YXofmdR$?Mrq4OzuddSJ0)s`c_@8Ci)Y z#?nhHnOC+fdKOfDBZHsyyq4UJ<3^k26rR|ouR7--*5(}`c2$zLk;6m@2&T63=a z&wlIRyx7QyLlJ`dzuTtX2s?6GvOT1C^}NF#pEcsOA8ftoQLLgoBlNEN@{}1m2k$$@ zY?!vVoYQ;hVq`z733$!lw4ft>!O=a1FWP*w&(=-!ob%7+<3?|xOR33g z|6Tel%U2SiS8j1(-`TA~)`zF=GFW)4VE(`BR<|s|1T`K-Tr#;oZRKO{k6(S~XS_^% z_{zR=%7)7;ESS$owXMuGx%8zm^02|yUblmHPh8>9>5r=mv0+yeIjuZ9{lS$_JN1@s zJG@^0)<(UZKV?==G>P9h%jn$J(}hwMVkSmef*>=Qo_nkbBS;l_*6*Int3$iB^7j)ma zJx4F9s_Xdl#b4fXU%RL&IHgm7p_q{P3t&!PmYg!>1W}P(@8F= zpZ7Vqw7SV&lwEdbqTO#*iZdyDm@=W|@zpUSM-{jJdb z*POy*vM+B}ggMGj(ox>|`^*cgz6&>hpHZBfDLVTSJI5Zws*mgV1u8sy7C*KNJij!_ zs-HE_t7X#38#}swPn#tB&H34^->io=ytLdIAHFgrx8-x= zUxkjy{U#d6H~rkDx-WIIJlm0K`E}e4yEk3dcsp;+{6iXBdfnKh?=A4#X7JWe6Y@ytFx5ba_www;0ZIGG{FfMy? z_oAd_*GX&NKAA$T34d$Vs!nfOaPi_&>$fMSE!Vs}>z8Kn)LPEl8gsXYB)|8VZ8Lq{ z8Sf*LBo95$@n^MJZ&qF{Wi+|%G_R8T+bK;_q3$yNwzsDEaBg2TU&!yB-@XYmRaEBm zWHL;#{F*U&K2Jaz@4?BemNvh3{G9vz$BW0`VmAK0pdZ^gTX1&HbnQapElVdXDvvqg z^J?i1BUN#QXN&B`nqlH*Uwq(Z9M7gIaOWd7Z;nJ1)0~FCSU_&lFl1cIMr1 zRF*s8DbgqWDO7-Mg%Wsps~3NmYt9cy(=k-LoiLwezNY=3DK;)0TR_u7$tw*!P-G z>794MuVa&AA3ji8`dVQ^z&dWjg%uYxa%~mVj!atC6`B(k81vKR;+eMQ&@GQl(r!sS za=YH6Yx$LToweK3E1nD@8(hm{SpyCP-r318^Klf{xtI;>lvZ1&m8l<}pOyRWAA@x% z?{wKIK^b3d^jb@po7R_?&z@Lq#A{hxt@Q5Pl$T{Mo0d;$+N)D#m6v>ekKKkjzw%9% zFsv>Nu?mdQwEK2T(&%`m=ed}teoJqibDEX;m9O1a<)Qd-k2OCZm*f}SVfdOmnRC;M z#lN<__TN_!A177%mg%B{^(_y(hBc~;Ngpq1*t=*s>@J_KCN8x1!mn*RpYZBmSjcjg zox${L;iurgy)7S}2+Xp~hQlB<_*eKEH5!-OXcx4}WJ*ov`t{I@3nGrrql} zo~ONh$2(hd_HzrP0;{JRW_T|xbux1bFRpN#P_jt$@U;ERCC zrAi@zb=_xgAC`HzPDAdrgX;M?R_$yno*3O(&b{Kv%<44zM$7w^dEc&{SkmwAsC>}* zgY?OrC1KC!?oYRW+Y{s0l)m$9kJQ#P%E>l+ zY+I679DjD6_4tJoSEH+0J?0!fFWoz*dtd4|$D7Al=kwcIJr6mh-x8#?&M3Oj_=$nm z^i7i|h-)oNVLzHN>3^PbLfp<%PtB$mabL{jlzJ3xfzL?rfcr^W)aqy(OrnfCF`FS)j%`euhHgA8ruCPLH zgSlJDPtD4m*=BWi3CCkE)J(n5bNyhSljC>my7wDw-u>8n_|xM{H=hMPJNEX{#_b>0 z{@c{M_)PEZrOEl{UxUQc4izoTQ91Sca+==Ky(je-R4ksDSbpS?zD3C!*_!MV7LOJD zgL)Q+9qn6KR~N%o!eeo;G%hVMzL?S1aq+9lGKQspQx{%STArI1yfWm}j!S0d)9ibu zIoz7|y;sfQnx*RHXYJXgMmh&2)$eRziEb-gd+yD8*~YH|Ej+A&Xly2rL}%N6>i-)HXk z-Ynr;yk1LZcIWpk@7WgjsrEXFv`^i+^CFLD@Ryh@nN49&EheQ$9OK!r<$HVJ@g}!O z6@ROxe`oG{Ipv(i3hPP!7Z;slT9N#5i+6+ZI{xL3R=1mXDJ$*}n(W;yws6ylofEFi zj$IgaO!m!o`6)&dRI1J`@!6=Aere~Fe%1VW%$-K%K|k%gao9ayi)Jzf!Qo@m^+tF%dGUFRB;8zwQ|C)sb$^OSPApSJFS zo3gXBgK)9uv|PQNAA-VfPMhu)DdpceBUeWB%1Legi#MHFax6g9g3fHMcQGGMEq}1- z4@2Mf#E`Gz5_T_b^h8hZ3yZmN&&hB56cZy4hPzWw7u>Qjp0y+PxP_If@UGgT^=!wF zc%OV1<1u&Eyv*wQch?xdWx75&bT;CQxANM>Wy;gvN}u03<+96$?HlaXn$DiD@L`!f zzwr5wn}1X+?lcQX22FL-Qw^`bnGv#p)l}tk;p3&>t&Z+lwg1NPqo!udfBnDUQE?#J z_1HhA`C3Nb#Ei~NKP_N!J6Q1Y%cDy(`+w!N@1BGhF?* zW5OPWqB-6>r{pLAQKm zUAl~dj`R2L0?+TAfB#;)(D2B!_lfmP&fhHla`Mb9?>W7;B>(A!zrj84s?VJI&r^Nl zaf|hvM57)K?+nXIX^!6twfnbj`!{<%|GU_~=JESq?ALER)^hLHdpBj<+y2pOdfl#; z?7y94S$sElqnS`^7}Fjj`8OH+>!W`tq{~eCe{RA?>E00YhnM-k#_ZBkZTcjq-n*)f zLq|PawKjJFZlGIrRMp4)`s^wtNH9- z`rWymF(vwc)Y~<0K6m}}lPJ!3uqi$DQe^essRw%9cHMd%yWpT*`HnZu3oicq;eB59 z`n~r>&9`GtrPqsRY`A{ZVR6xfmikYu&bwZ*f8Qtb^KyijQ77ZsxF?BojC8e%7;NQ= zV!v;G!Wws8?FQpkFZGJQOEtB;#Pc0f*|vP#I#1X$LossU)X!74JBqxS#r80*>1Ge_ zVrS>)0WZ8iYp?rpYRLyD$5Ux3@&-;-+YfJCsI^W!+;v$l@2VXyzXrQY+>4vAX`8^u zn>=sbgDMW^Kb|l{BS|LKvviHY8e{R5N5zxpysdSgdgaK4U%4`@-16LY@81|}dR%UP z^r46QdUnZi#lRr1PuE=+rBnsHv1UK>sH!Wcj_u|1!WMIuv*D^& zX9Rz-i~6;!e%6AE3tVS)zb?J~U47}2XF-Y9m-Nz-wr?&A(oeJ7SR2>baxtg;#fO+* zSwA0saMtqJdq=x1YxU+Q6E>gCT{Fq~oY~~EK-S(Vy&fHN^*c1B7~Ol4Ro<&i;J5bv zbHiNpqWwloEf29{S|5|&^R0P1!LM<_n(!d&o&Oi?we0xY8}D&q*`&oQR>Vcd^1NNo zX1V62xs+9rWnWaq>D5*1YMBG`GjT;{fe05fI#nOwrRC=@cCtk6*rL^2u$0_N=vBt;E@}k+?TfRNC z%gaz)STU_}kwdy*dv$2yWOc)-PPW&s&c8U#Z^KD-tDAOQ3d+krpP1Rb?pTT3&2S#W z-7|yM{Mh*RzklhWYeydjB@Zj%)d%cY8i= z`c=ENSw6MoyCc(c@mYaaEwc179%Rb=dGX|g`|NFoaVa|bZ?8I@w4L?Id2v_bzFC`i zKP_U;i> zjJ@3Xg2vl{zu!(zmS^qcwd@OP&g3pWBX@p{*6G5KGXJ<%hqu~p zm#RbWZ?xB+QZw^vxlG*x;lc}-?#*AR)1EvhKR5D%$6lFNMJL`FT$sk%c}Mn6&&nsa z)IVySHuK?=3KVj3-gc~6=}fEY+`E#gN9|r;$=$8 zMIQQFy_mPa`B%fm_{;aN>b?|t5Il2Mo8%7XuPl~*tE|~|lYZULnc}m4wci8p&$rnY z&2h?I({}5FkkO=cr?1<@KYcyA`atK++g4M#?berMR`V9DUw{4joy)$-8w&Z?=bxST z`s>=81>(QfOkQyD#IFrmJSoO{+kRfliQ$-OwRuac@%7iyo1Z7`kF@_Dd2MyG*~t&m z>Gf{WAEaleJ3LbVqc;1a-M86>e~!h8o3urW-;reBxAkYvtouR=A79_x^4#nC-<(?! z)4tf0n=N2gs<#U^J6~cRw&Jkd@}AbU8{gYFXj~3VJ$C!zywFXHMHFLt>*`Lu@_3n5 z@Y1?H`A*Nv6`K-yuYdlrFL>p-)q%Z>R~N)*JXN$&R*;PM0L=UM#G(#Un4Wmn2q^n5$n9o zoL@FRRh`o4eIcVY@ehaL^1a!)inVVNZxoyP6)xDDZ8CMq&kVg1E`!*n-3D_ z&ZS<&)w|+{uSH)SixD%00`O&Lx*qP~?$y6!8IdmeM@ zUHjg9i^{8oW-?#y*yhjOdf?Dbo!C|T^!c|Lhc-6Koa6UQJsp3#dGE!V63aZEq{4`0 z=2>DB?pK7hzId#q@LuNVjxff`^wm%}?KUH-t>r$SAQ;T*p)YJDc4h{hQ^s zr@;%$6VguKzTM;X#W?z&U0KTA^E_qM^(yvbPc$ard|>CUx1Rn_%edqrb8 ztTJy-Z+9z5yW4_+AToV^|I-z~;#A$Ee%UsJi{cZbC#!qPl z=O&)zSI*zjzus1Sg1KdG1up<4SLH)U1MHPDe z7nAy(-43l0`*2C)f!LvIraMkI$sU||V!GSS$dXz$z073ytBk9)UuH#Lb3eY?`;v#~ z^pC!_RW`=^I=RjBA8TYQpFecg%=*oD3(pe!2iaGJH}KaUQ#{WfS?JSbZ}$F#Wzfm3 z1#|amrYLQB*u~rzQj~r-S0?k?m2JD;TNN|zFe%mgd`9xo`Mdk(6f;cQ&DFW;l+g66 zr?1_VpCNT8{`5bU8&@6}UD|fSA^f38llI<|hkgm_XqQS?GWc%zlh~W$xVX^U(Qm2J zhJu+VS3X%Kmoa(ss!tLN&8*}vSXlNMxrzNd87Zn4VV@bY&qm;3%+Iry0Sb?g7%yBj zQ=30Qd1C7kfiqwF8ykH!Mc-T3UuMW@yKll!-OtT)GwzN%lWePZ&heQ2OlvP)|NXVQ z`tz~Wznj$7ypg5XuWilhyti)3 zrhQYw@80TlyZX^{b>7=!{Y`1{+oz=*UboP}IN|g|_8+%T@6}rIc=HsM&r&XXl6X=I zjvWkLdGM*ki`y$zZZ2;vxWC0DN5N!~RWM@_!@bnf12>NSY)bNFJ;CN=f2;Sb-=2%f z(!F2aFhy8@HI&}X`?hVi+)eT1N1N&wzs!;`4XuF){KW63v`V>xvKr-e_BSu4 zwmwL<-|z6&i}#M`)tfStY&L7Js+}2F;Zaj7pW5!#`7ZsiXP@%?+v>69D#7bs?R%Lo zw`8yBhb?;gn`g!Z-emnR?R)w0if{b=+i&ezy4B`h)@-&6UB|`O`mzmeZ#`=?TJk!? zed>2%zaNFKjI*@v%()bMb^D>f^LInvm@nwtvupJV-I6^k;@A9rpY`6UkdZEPqt_&ckNi=*Vg%j+hE^Lc*Gs`C}@=Qwe_$#!OT zd+7z0UKX<>OOE{%opZ2ml|BEvX35C>fV1-Ma1QHRj=z|GH{6mMkq<-QJD79X{>ZEj&f#2lrZ_n&p*ie>-2P?3oW&p*7|CP%JF zX>x6xMx1lj`O1S!e(%1Upqj3ilD_Qo<4ZgDCO?!lmufw>v-{Hxak;BW%U|EWcbI2} z?CyWLiLBqtMND=@<;1$p{#$Bz$?bxMvGJuC8MzIu=d%@VJ^w1zaqd>*qK6+EYU5UD z&bG99ux9lgMV9vYRqXzd`2IWj8x*c;mw&&#@U8N!C#RlI>P#nOEo>e#lGKJogT=zaZ>*FP47F>&7$XSH9qoc z$;H;ZjQSwk{g?ZX32(mLakzN#-;>LGCYA5oc3Z9V+su325q>j2WIvqy?`zJlwn)>D z0U4JJF5fVhVoj^RQX-x3&!u`5=d;SiwpWi#SvV`byP_lBMOmEtc4fDi-&BkBzLxt774*-pf0oNHAN?+D`r3M{ zWv8y>_k`TLw0hl)6w|rpiZR-3k-q8zl3%pTcJwM|He@~7kzjgvrsB5Hh()vNOt#c1 zuHGl}_=G-p?G@JEb+x^5Qy)Hw^kn|B>de8r@_Da?R_(UW$hCNDSS_=qZf4osmkGQX z^%};sYAe3xH|>v4+v;WAqI=`hjOOgsllXIacCfE}HczJN;arm5no?SBTrRRC7^kyDtx4JD8TY9Y(L~VV~fBojOf@RLlR<|#=ZFzhr<+Hx@ zrF-++X59<_vXA>+wPwaXrPEn@Q}sJ`9{aE_b=~H(51p%HJub~OU6NBL5P9y-hGR3e z7g+W^a2NfVVZZixalOcnb(7}K?EXGi?N0B*x;T$ZGaL8!y6qHy)s$`Nt=^jY%I&=Y zmqoZ(#hIIP%kExVwb!~g;#_UTq}JVWY2OtY^87X}?ujeeJH=#`aayQ$c2#?neJaZ=T@%x?TevbeCC#Iv=x_1F0vH-y=8}?TDquZ z@zi~*3T~=*txDKDujB7p<*b-PSMS)q+461Doy22zn~N6*&NMCO_xtVP-J$J#eBXyV zpEvE4V%omh#&h2sHr3;gWLb7a>OH>5$)4R;T6kyMuG(Akt(=Y-{xK52b(Lj$a!`00 z%e8%C9{VQVsWDqL@y?0D3(rj|wl7+=b;1+3rB`FNC+7Ay^G&{-vF-7x4BabjzZF9B zbW5~MPOY5qzHj=suaESmG`UH>XuWoCvpV;Vz*Nq6qH8Py)_r*B)B4UiM)Qc+={qe! zmptM*QyOP@r!sxr(X{uR+2ZC*(NABL*S7WTtNwj?-Ii0GH{35eMJ)A>n|0r3lh{7L zbLN}o{aErwLGXl4{5$T=EZl30Lm#d#+hKm>wuMplau$tCJYUw7{g~+z|3bZU(?*Z` zJsz4RHxw_LpMI40_{4)uzH*z>Iv>O@DA(Mw{kF*~ZXRd#45@Jc_FlJxwYRwfti9Vi zP0vm(e`%D^S7v(I_#w;l#IBCqFg}GDN={lrnGIfWOVOzq|ikuIL_O6$q%GpxN{A3lrDo@%50y0fKF zIQzQaqTRn&Y(D$-o!zT9#k+U+Tm8QL?%m>VqRxh`k^c&W?Jg;rK4R**HJ{bXY+KQ} zrAkd7M9!{Q7|Sd$Eh_8X^qyDxcTOcM2YLo%NQCa0s?4cXw7ox9B6ID6bGZ(i#FQx$0b_!vpifHAkKE;S?VwE#no(Y^mCmKtrX)-6b&c} zoBE+{p2@D4W^229Y%)3Fb7)%81Eo$Tz3T_-4Bd$)-(D(td+%9z zO7@nx-qCvpn&MvX-7UW6%7y~XbMH)2{{EOgvqyPS@xN5P51EBq=jO_EE|gmE^7+w# z8|t^@9hEmmtO&bvtn`)5rFu7YsUREH?k|U3XRV(0L{@W^==m<2Z;=b4KCU_O`O=A1 zawqC%idMd{U9+UNt%NJRlJ}vIs_xcZHq)0KyBOM%5xe}^k_rA3f6up0o=|H0d;aaF zor^A4iArt#^03e3_k3%2*L;TwhdWkENvb{-%0HPO+ICL;v45m%{zQgOZ3od@ws+>6 z1(5P1DX=Es^dm zac10O*vtM|C2slyv%W*UOIyEhx-RM|QMmVUu0-gIo$IQOp9|U0!fJg>xuV)??%T$E zjsHgCGMC@fmORT!td!qgXxGK zPqp2Usbaq?ZOb$FpAVEYpUi2?kN-L4QO@;e8|LI6e{I&ICCU3a$g{NpntnwMeq$$}-b=+G}#^q8DvS|7b2Hv8l-|^9_e- zK`2A@(+y#_Rvdmh)6ZmyU~HYl)#6JvsvWTp6USZ`J&if|Rq)S>n(1@G8C8F0iNwgC+gR{S>2JB@ zf^RW9k{0jGXZ2!NG3=|+_Ug{k+uF|1z2wH8*JlzY-v72Cd~17F{V`sFO{^h%P3(7> zH9T|QCu(}#QD})0ws%mH6b{ zwcz46;Ro6aciAxO2l;*ceDzf7p-h>1)^ju8uj30%j9&hswEm6El#fgGejMxP^-(`u zo2F|nd_`yhyZ_>&EH`u5CfV|Rki5KMW8;GhQ!O`FK8~Cv#52V{^GeK5&UrIDLQ4)u z9u$^-@GmOR?EF{Gc{dLHy;V1VHLL0LLzlluhdf#%>+oDMVdHsjt~E;nPjuC=C+0jb z<(%_R`)t7D7i+$6^D4YqE?T=jUHy2kn-nAefxGw8r}X(g+8m&1_1EmUPmNUagXQn? z{6ppM-i@oOEh+zes(6q4C6BrUTVtg|`&J1v*Q>tYJVW9~#NJXy=KV>AW&$!6Y44v;%o1IpNpUf`1&<>uw>Jc7=$)VF(|hsN$<1H#E><4B`1^R}%s^3cQPIyvqr*El@!41O-kAFOgY&h^YixF%U1~Rb<1L4LlS|Hb^=0U2YwhLhwG!ndG>XNIuC#s(j_1T^nBCYd+?_l(m)hBPvW@dO%(w%q5 zBXUCi_7%q0nT58N3+(-$w$%T3)6t^pjc>c9Ui0SMJr<~Nag&09Jdg8w|# zFD)$G->o)n7ij&EF0YfWwP2RStA`tWW)*lWuL-IO_SG)E@NjwSRI3*`iTfWa@Y(-u zYqmJA4M*d||Lom6|5XQ5%K^#bwHwT&ZqAyuAaNV$&9%m24N@{7Uob()W8W?+Pjpm6v*~ zUiNX8*K)a+9HzVaUv?*M?Eiao!;M!Sy(Uu^1Qt(}ySV=K1NW>wJ^m{nEu2z3QAyDK z*8xu+hs_vlqVp2}^u?p?^Y+%)ku3j#mLR&ivV z{AV^<=vMNV?UfhI$`egEPG0hG_t@ClulwclN0ID(UoY#Xe(*hhJ0s*&+%HRxE%ray z4(FJhx*k34e8u94R+W45g4aY_+cO$Zw-;wlW9j|$^Ozhs)M}+IsFE!=3Z4QFFF_-#wvwbGxi2_v0C_SIY!XG|`e?ziQ{cQ-&u? zGv^n^r&ew+tt|(S9T%P-`|K`yCGy4Pd(LGX+)|tc zEzUcN4Fi6!D+(0RV|(?+SRnkel<|8Bale(%W27R2!}F7$>pU}?9AqbpD$BNNlS1H!zXi9Gp7pwK?~R;yIa2gN z$kUXFuPS}pb<^sWtk~hj^if1gzP3BP)Gz%hgPVuc%Oh+@;xDw+f4aZiWZ%1v-*b9j zpARbj`dUzY=bE~|u~qLCud^OHb$82)r588u-|czPd-3T-i=#F#2&{e5KFeGDK$rHh zT^CpExVKfydA~_~T$*3vZI3Ct->f_LT<7n~bK85BT_2TxH{2BI*1P6s%q^Y$OTTX` zyi!~+{a*18#aIhYskc14r6qHVS3XZzlh@C8entP8ue0Xv)Vig;>`AP1iqCnmo^PVv zZ}Ygfd|WK0y)f=`>6?XBRRK@uL_fZ9-Joyx(Q{^ftLmqQ_h$aSBRu!F$m4JS9D2^r ziv1SL`mONVn_C$ne^+nJDzhwJ$b3rGPX8(I;>{-xKbJHuunf&KjprE zf=6rByiF_gd%3>aqz0Nxi3;sJy@*e;#j51%%u|0Zy?E48de!pgv!ENL9A_l@3PYIQ z-o12P?s@LZ-4{LP^1iK#JihHZ-`~ox2RDBFJh0S#on6xV^J*3M3#a^>lh2s?smAO)tOTv!+_e-nS zeUpeaQ(9#2Hr@7mMUUQZorF7^=AW|*uIs<)zx|?y{gu1jS0&lmeEjbPE}0>|tvEUI z+CGoMM3=cMKW)`Yx_Nu@1%{GaP_borpJ`0jLx9ij0^sD zu;jh4Ygm||NxW)Jdbv!xOy0|?OBoj&qHlU{ThzF0opH5i?!vhTh1chitkJ8kUd2^+uIT_ z-s_-s&vw7=*;{t)kVey3VYuuELT!n~;6)^j2p~3*`{LmY;JrRgzJQ?@ZofjsITJAHtq%ocJ~OL)dc< z_o%tT=2C*yHP!~VZXaHC>&vFu*C&*2opkrSY}JiVn`ZxRPWf}w^5Vae3#y9K1NUf8 zD~{CkIA_tJ!&z6ds5X}K_qHF7>@A(!?`BNtGZo9b9$D?b#G38DPE}~JW^H*)*4ter z65oDKFYR4;&Z)d;&+dr4#~!iFR#kRK_q^iiD3)J-B1H1#$B*qZZHn35j_Fw2q+4t{ zqr<*H^Pc{%>#AbR zKmM1g`QY*5zWVbQ4EPgr4ma?A+hgA|O{qA&Wy+=Nz877;hOgpieKcv8j_1el1@ZOb zFSpxWUSI#S|Ng(%=6{wPJ!1dkef{6J|Ns2*|MTIp_T{+xAG;&m8EgNv#{d64zkaR# zpYQwsfBXJlrta_W|BDlYmIS)x>`z+3>-<-`zjyu>lS6+ehi{s4=z;t>v3d1U1{Z%Z z`+k{zzy9~{{jKpE|NZ)VzpnecSJI*Vem}3j|Nrs!e~!Xw^*_G;{{Q#w`u)*QmgYQB z{M&kR{{D4Ojf+-U`)W2Vd+Prt^^Hj>*DN_Z-wRXfCz(7Ib~=CDBJe`bmy@a6lIPTm zm`r{)tMHwPqPJ93_}t^SvaK?MPRticvVXR!N$r@w?#~&k%RGbaZI3-VX`mWaefI8T zf3Lrj-M#141TEWn$>Mu9$1JDfuLbW~pX^-b-)UK8N*7021w>c!m3+y3fLN~t@< zb2D}4cmI>(Pm%hl& z>@x}1J^R@5_|KWCW&O|R98*90)WAn~-gUgE@`2#-d6)kgLd};C%uga1|AD7QMQe^!;Ow>fR#lPd= z-8DkT)UUrObPn&Ix3h5GQ-zG6zde&Sm+$jhboSL(o8`V29whv6arSSMaX%ZXaB$+@ zxf)X|);&5Y;2)7`a^2_h)8&`LIA(n?_jE4a8n8!Kw|SfP*N1bj7#%pjGyW5=_GG_^ zcV};Z^KwuA^0IE)iz#(4j)~^q&sMP%|35jV(5&r;3b*C;7`rc?o$oAKuU&n%v+U!} zwWc*`J^yaK{!vL7Dei4{dHJ^Fkn^nLo4P7oUMl{(u)OMx zNz`<;rCvRYMbSBDWUJ( zs)Ur2iA%28ExghGZWE8MRnHSyr)x{J?LD4ZSWd93oOe(qj8E0FC(1@M{YJ#;xQ#bY zrhH#wvUfGB?#}JM)SN!Io_w0y`mOnhTg~rcw?DpepC;sAzTWZw(h2{YX-CXoYq($9 z=KGh$5%kzn9-MrP)fcB}NQ<)0Wp%WJESSyV~}C&DVeET?0v*(9RZ5f-cZU39D^t$zh!q@W`FI+mmJ#rSi z<$=S2tG9oWdGPSWJs-Q)vWMn8c(~$Xs`}x>itjdVDK*~1`bh3{ z`-TID4VRY(81wntKcCcYaQUI))vdc9hkUqvSN80ZYs(+b=P_Dmr+YlOt;qZ3+nH;3 z$YA)BBU>J$LFqe`4{DPl|JAe!2Tu#_Q*W z$HCXz?>?IuH(w!R%khS1vkuD-v5MY<$HhC zrIz30_SmjnV*1@;_i6r{)04k^UgP!NW#QKTDSLPO-1@nE`J*}(?YGq#kKUaqe89)? z{BGbq&MNb&ujh-8?mPcx-^{6B?i&30?V82^a?{(I*DUr0>?aDJ?7qmfR{Q7oy+>W< zs~PrfnIjzez;ajm8r3zmJM!~Sr!-D~ux{^Dzx&HR&r@5da4`GA?$z=KGX-ASRJ$(@ z6*^xOaO1IwRZ?utuBE$M=Jl#%OyPKWQ^C-z{N)F=UctL>7b%-%=j!`6y**NKby2+m zQ`^_T?vj_U{1)t$S+Y}K|M-SH^J=+i(;qJkyk+ZD`)88P8ISEuIt@>sN?co;a@|>1 z{e0}D179ETZH{C(e9N&y&Z+P2Y_2e;IiKfvOC9CBcpxISPN=i;Lvr%x$qVlufAeXw z=gyAD67HFIzDO)j_2hY7q&}w`L?6rj*O_>C*_B^!O*TA#^|efD#`7uv+CsiRcz;wO zwbD|o?JH9bAJ^v-%Q@FO)5h-Ge6#J?~4NxDf?>A-TJ*lrsHNw40HR1pWkv#F8O+^8+b}RwU#~In9)69 zpWSrki)oSuEuxpUyUsY_6ruPxSMTYe6X72BXLjp^mWkZF;O7EA} z3l}5IcGVU>IIOYPWqRA2(+9t;(@lRMkmS1Hh~280a??^z-?}}o`o|L$Io_RRL2B&_ zF8<@(W%anmWJ-Ng&IP5+DfLP#cW$Y7TJQZ^{Z#7W+s`*l(A|_%5^bmY%|C&sO?UpJ zQ>~XbJZ1jx^7?tUhU~@<{w|)%f4NHPPUz&!<7v#0=sC3EeAddhpV%!vWtxb~)`Z)o zoYt#v`0H}8UC`oF{`{*>^2-i*uvFiF_YGKNM?8e`0oj{%A5w@3%x2)9IVm zzU)2aYjggRoBGjZ>>q9Z`(ARed#91|R_}d_?o+Rf_b|2l9y`Zrfx6e%c$NyMX)E*h>#V&H6FJ9u zXIW6?>?J=}mlT(n`u&-2zk}!2ZT4H+&Hu#&82#V5Af;}Oqfl)?e(BKdPlZI zzq)65Mo{s1fgJBjxy$UEizP11?Cv|prG6{>{D-jKC!c-z(={KJak8hd&1yS5`&>-# zHM^=WeKl+iCTAb@H5VlHv)gO2SPAGA_p@zzb1bm2S9axA)&&dC7mCaiy}011--gA( zB__`Viam9$@4a}_>y~-bBwXsjcQIYsrP6zdH@s@jXHDmo7nw`+js@P* zSai|;Z0m|iGrY53Z&|u@*X=lg-hYYOFQN^z*52c-Vd}db!WA_q|LG0BmO_J#yNf}D z_{`+ar+eObt!6d)+dgw2Lu$I#uP1ZNLws8fOp$P1`OxyOO>kb4TBe>;puc)6&$UUj z;$Hu((tN!5-h|h=GMRN-q>FOp_utAtu~j*d#k9CM-jOfso8;c~_U2i8mrT!2T~%7T zPxiO`lw-H!|FKU!w!S)c=Yi~fu@BM&)F$mM*?HjYjiA}vx$p2CDzLg$lz;o9!4qND zX&sVZB37o=9C*Xv-7u|9VX7_P@>x&I*2Ny$ptN>1fB9R^olcr^+1JlSvh+oLd$KvR zy3C+`-8BIdDFyH1rDC?0%iFe?9o)QxaYN{bH5(4zSZ_6Jw|Gw!?{c2d0#1cnF?+Ti z{(MWdH^fjW=;M#*WVYQ#3g=bT^Zktt&hymYJvQ-=!U~NHlS4Uov|btag;bsjd7N=2 zJgOr$Ju-Cjnug*pap$h=K3()X>+pGtOO2;rHvhT0X*<(}!*+Kc&v`!g+xCkbn`I>v zvSe1aO?W6~8GP8Ttzgx(s5`IYgw7siFpjn>Sf#gMR&A+=naTAv_XL|~Z7X7)Tlk8% zV)OdyO!Eh4>*_4_YR^BvFzCC7b$^rF>x3n1IOc6N-m-G%lwX$3QM+e#Gd9|<$@beS zZspr^<$QtNXCv-KZjW}(Ilt$U>4x{8YmBz`xfU|C5-`Ue|HdMCFy8q~7%XE2x zk6YWs3x8F)>{NbP@6-Qdl6BP)=j7Ld(!DF)MI@ipUsRTpitfMh%;;-Ju1s+MhU1lA zI(m;?QGT=4zUs)a{*~@GK2?6Qzx z=?B?c@7hZDPIpSGdhJ_e|k#e>pFtqFSlC_s!Y|K1Fr==I*@owM)jRPRC*7UbxeKrBWL$}veCB_^-Rl$f#mra9*y7Exqu`@x$iI0j zo&~*|D6(!%Y4zJRSqAS8-_tL9UT~yD+I^+e!>$>(jMlV16Eew)J+R=pMeMxWZ{;@5 zzpAc!H&XoEv7e3?@6B?U*7tv5O~Nbw-ZO_+SxBAzuc7?mR(A2!7PjyQHtTnvi;@@D ziIFw5c9?l@+6y*~)APB~r%f+N`!OZ{d%ebjEq~v~x^b?WBiG&cMKw*bq$St>N9n&y z8Yh%)ep~#Z%+|yxV#b6Y^?i$(=KOf@t?YwxK5N1>HTqV#Y_$U)@*{{c^i~-GZz=D|2n$ zIWJyx%H!z_Q#lhOD_{RN9)D%r6U&&-wo2S~p8Uk(%aJg)tJZ%~66z;z`hH=>*~RK+ zea$YfE^ho18v8);;>K!)`h@|xhdnqiF-xn9EbNu8?0wM~J9B$qiASY0v*XiTnbJEA ze{3^Ll8nxV9xGVttgmzT#*2@07<-(}l)CGY2y z%_j3bUTQ7m%l8(XF@KWVETttSg;8q{7wLvatv&4L{e|(>x*5UjJ!W%~>NHa?u=K9u zaw|41+!>M}S3GyRccQe~vrm~5^wt~;3_a*!mvi)XG*|w+Q#0>oOzA9rT=iDDEhd34 z<<@(~q<67=i`Q(ZI9T}ht<$XcO^>Y}Wp2@NTztCnVvOpp+pRllB!3EQ{`7;-Ave0{ zI_nSZ>z_Am=bm@#)Ym;ZhKuKgCx1Q=C+?hbG;WVmO0wPk{7fmv0})5R1~vZtleu~K zbrHGy2Ln^1?Upj!yXpVa<8sHjV@(aW*4y~3i8@tMnfKRAxAd8kX_Z06WxMknXO>6Y z+1%9>clykYV-x3ntz{B1E2i-|$*4(3c z{>@2A*Zr?LuI^s0@k_LB$;>R7xzBEUYMwnjxp=9$qG8s>hcl;~oEr7&gY)?s5tCbW zi*{C(&M#nF$0fyT!OJjt-uKwsZYi+^A1#0HTl^~A`|eeSgygT0=L(KK(qFTvj{pUMY?pm$1RX?n7cZlcuA17b#S!(Dp>)XkVzb`4?oshSxdbiZeHs!hI zk!Pv+3AXN<$f86`nSn0Pd~Dqe|u*8mkV)xo%!2Rf)_6H zzqKlD+r<+_dks8U&aU<}I%PS1i~7rAQAvZ}7hZ%M_WbkV^8B^DT_^O?kGwh7-_w8L zd6w!&t3MYW#!PKDlMDp?MNqhBv-4V8La#M3smM@eHQaj&a_%zic_4THLbq|e} z>%CI7b8lg++MoqWD(g?+nlWO_`d;Z(g9Q}kaPc2=Itt+C=r zgZw(atX0c+LipH`fJBpRxE>}NZ^-w)0&!Tg~^p{1= zDjO>7Bvg|EvvSAeMasu7eq3t3yinTR zG4fUD;}c?unhz>V-mf~mp>vX(q?yTeBct4TA7#$egp$nH`Yi@F}}_r zUUYLJKhJETJJTf#9*GKX{yw*NTFJh|tT{K<|8kY?HDO5D;*$R~XOqE?vc(f8u3;)( z8{%)Z^+1gGJvWBK-eUFN__A_e8ngEL9bde%>PzdLiEia_L8@~muJ`Cwmb-O9bmd0# z)vX1tS#J*e96ObpI&(UcYx|-F7sXCrKV@|Np3qs*^M}{`SZKbl|JI$IFM8*P%7jeK zYTvT&joiktePPp93LQ@KY;x-*-)h|Ap^>H=CsD{W*`r#}%f6C|XC4Y8b z`mbTCEc#kD`A@RPQT?5h)uK~xeA?g^KI@J1V(s#U-z-)?zg$%ou-n{#t5Di~>2@Xk zsrzqyvshxjt@N~Xx_L z^iow9dHR2!*5?tO@@2u^FEM%oi#C6%dVFtg;;Y`n+x1=Z%HtX~7m4_uRkvStRxn{* zOvN?3vnOUt^eC1;o5k?g^gUA%+Zu(yLbF+#>ES0>cPuT~=O=Vh;}uj7t?TR-HnBsPp>XW-@z_AEnnj8l!aLrCflp_7VFJq zy`A@G;pVbUHQ~MC)z<~jXHRld2%E51>v*N^$=i3UgVj4N`M+Pj&!pQe{pChd-o|zB ztiOfmo?HF9?`)xWW$FEm{Kd{|(+qpwv4+&mdzJU)vCga8FOJ_UUUzf1PSodz58dpw zuVvP$hAozlH_x!W7w8qNpQU^!@KpPJ`AxHqHh*u=dTS<~8h(L?BgV{_??iz2w-4sV zWj{@LZatf{&?>|-k+))T;Me?Pt#Tz5ZpHl*&K0HqNMGOGZ)?k z-`qa4@qK=3eDzb!7Hy3iXI8uozFEFFQ0>Uwz9VVuwe8a$Cp>ol`XMJgX{s^b-rsY1 z|K-Yr+6r&;DE5~yS5LZpVj}-6;gf|bym#+j%}!Yy`0dmlo;vrox9cL_^jv=5=Jjo@ z2lHOHS0DQLuitvuw4!fLU~4D$qP#yE9`YyMFRfr%yYktkr^fH4oW9+<$g|~};NGpp zi_eR6ZU1)l@QU5(Q)f3`tLhO>Nh{rVTQ9fRb6()8&fC*0Y{akpTX}JEqFa)jvw`eS zhnGChxtD%y{@fI9W#aKz$Yj?MwVdPc-fP}!Ik7WjmH3;}@8&uCebsudt^aj~&HHJ( z+vYSH#+dfL*}hKUaDTQ`u8CIi9GzWav64!z*<4b&Hx4|oeSRr3#w1)k|9!Ocg%_K9 zT0UO7v!0bl$g(Gkp~yV2aEIeQi`$#dg{-hWS9$feh1Q9(>4}^CZPeono#y^sRkTs| zU_al2*tX5aqCLFpiu3eWKihqHwXTzd%Fd4C0pFxFSM-FeKF#05`)=2(56ydQK-A?= zQ?=9Q%QdiHwYa1pS^hn&eSYEdJC=XW+&)pVt>m6q3_!W~cQw08V%59pu5+OWoE zrPE3W8Kb9uzCKr1o%!0e_LjE$1l>7OO-4}{B;5Fv4a??CxqfH*hQgqGJ?}0_N4{)w zleAy*{+p6spUC+gv-hpoS>x^G`@2~+k!{=jYm23OnHJaTavF2}+VcJOv56)ef+q?? z7G%aT+NSIM=&E1jC~WSynEU>6ZG(){>IeDHJlK)u*17Qh`h>e?eOKazxkQgV`ZPIk zmyvw2X02eaEBk#GVcjFKmO?Uo(!D(Wcb@8QPxTaj5O_M*K*~$8sOPL8zxlh4<1vw< z`}RkD&y~^c{hfFBxyOgwyUyFSYGlQ$_Fmc%m}@-Wdew)F;Pa}-9eZUh`?{2TY^Q&j zY%jIF(&)mT^?GOTF4Wx4n()Yfhu5+Nfj7(}>XP(i-5r)*oSYuJnDft)$?2hsId9dL z<{O3I=GnXB&jNFv$N$XgQj$cn@4WLnv-{H#v#v`^1-O@}$5kmQ z>*shY^?Z&E^s9VhrT*jArNr9be?ne7@}4CdyY^qJ?8<-JtzRU|^XOeT!rCD-t$Ob+ z{Tt_H#N&k?zsO=QS5*G3G`&*XXjA37j31iEE?#|gyOn24YD99ZqT274P?M;bu1#n8 z4!O%DZJK_R_xllxQ;+{?y!*CE?&#fl-QHX0J=T~P$n4XzShT!l^O0RkCI$$uI(2WI z@`MY|JQq(YmAiW{ZTbDLCU3YoX54*sJtO{!)%&{MZ_5`=x;b5?*Gq0!_u^a8jZYNy zuI}2Xb1~!2rH#9c3*C>`c}2c@yX1De)!pUo_ol4QSh+XagD1+>dv2lJ(|_KJtys40 z-qo%wD-!thl_@Z{!!qWFGrdE=!7h>9PX5Rk(?Z}@51{)5Im-qAw z?tbX})+*M5f?XRA%>GTto2e zHMu=|?#JL~EG{sHn;dEpJM`QB zTKnx5r5`SFN=F+k|Il>1{0uIraDx_qTP%0-2H8+bWLT z-EhaQg~#R58pZUt_qEpZoxiWO-pu}v(wxoxZ+1$oIpU!Edqq)y!8y)N{jZAtwL9Cb z5|LSTRA=hlDaCpWSsPE~oo7{$o<61c8tZd=b-tU`jKWD%(@U@2yA(R_>P+rm4lCYm zdhqb+CeCwZ3wNtrx4w}PGWB}FRhJtXB8xKPOa*fUi>6+`Vmoc;iEL|mUf#^C@SqE3 zSEDChw~Nh7=qw1Duvd_^>QT_mw<}MDyKb8vnlE}YZTY>ekLFC>zB_Er;iCQ7M=gsP zyGk}K&JyWapO(FzOJm#9Tice;D%URh9oeWDw77y#_)rzL^cbVs`JayOhz`vrxR5QIv?{)T0y0`0#uW<7}kxS)2GUM-W+pD9u zZT_Cw=ceAYxPQFvN!Z%jk5U@UO7TDKzP(J;2%51j>qvk~-s4n#S+*liHMS<3f@GqS zu3o&u&!@$`MvMR3v_mZsp>l6@b6;$@aPIUvXO4!%)bFWmT^U@n=5P3y611n|l>7Vb z(r^ABzwkT$(*zNL7pslRU-NA0wwG7QU#TK)_FU4f^2bCCg$xnC6{lC4@Vq!Dd?Vvt z6T=+!4;OA!GD-IyVw~8wYfDY_%M+>()}KAxcjT$^)P&5RcdeFcPh|)dH!EAR>a=UW zztmjsfORTW>3Ts2!704N6XX3^kB$3QRs9K-KKuBLuf#5MVXmUJR!44YY$_|- z^;+Qe?xQnVJY^~*CpWD3yX287@$YZgo$52Eetzp1}TNhXC9{Dj& zFRIu*@^jp}tLZXVn;&*}R%D30dY0$=^hD(`?Z6so_gzy9jh@7KS-;PEdupvrenh9H zT#B%dWe*oqO`;5cFBj7t1HRx>b2K(2$UF;Ym@!)`Go_vD+PQY^RK4y!dk%iPbo=`0 zTyIG)iS2A<*N&du(p2p|-!Jmp@0Kgyv@%Mvs&~Aache{(rEkGS+jW<&Ub=2)%OP@N zXUWSaMOPy>U*ESYX72oG$&b;o>vrv49%EIhdEe~(9q!jM?(U5Hy!O~$-<~{8`S~~2 z6*qo=)%durX5ZZnGPCl{b<*`0EL8u$D?DXZ&u7z+Ti>_u+f=J>rhBi(Y(l_>SGKp) z=WVUnRX=8iAKmZan1527ziVGne021gH;?1_KtKsbD@bd4M8>YX@d*0Jl7|L0f@rm2cURP$qT&+2qq}*R# zezS0Xdxc1tMt$tfvR6evcWruGdhkQ^uf1oBUC$MIzl*selRM|Q`>Q({PMbHpD~tQ| zV^QLpozKKglb#2i*OxlxH-k6ceCynoT-U9xZ{PHt`_}I#zrwy5o;gsao6>13UT{(M zr_FPHx4S+f&PpFvX>7gdkt>;=ELN{qVizfqqb~fT`@7kBt#bF5&-VsgW7)THp-T9< zzubqmeO}u*bGixFYq_fyQHS>%KIFIdI#{3RZW=jzy8oSX(!KgJtGjk@P(Aj+LhohX z$K87aybiw4nsZ$)`)7^dle|LhJ7xQHZvC%Yu6BF*glV;=0UPs+US9an{#)rneElt+ zcijOOMBFdD$l4Vz(Du`5t=)FT%I$wMld|7e_lh65bx!=>w+&m%nR5N7`QPzmUDq}9 zx?0ZN=ihHXzc%yLtK#f>?}iUM72Z6{GpcHj-g)D@Olj@vC4qDEF7lR}l|SS?Fe&(K zz~0J*x4ETE#Y=vNttw{7UQC-diu?GOql6 zHE6FgznQ(q=U9!h&u%79I`n5BJ?EqMc3b3|QpNt7%3o(6aTzcivME)LWz>$@`RvvpbPwr%0hK%0K?x{!SaZ^K3PH~TQC@9uxIYy0lU;cr9N{B(0NFYXdxj+pDR zYv%e7PngP$>zjIhuU1)ae6B=v3rAAzPn`P;RPS_)IKV#En~bQ`!eDBPM!(2;vv;; z#W!aE3;D6p{L9J@$@xcuA0FQq9kZ|9^mngYW9+X`_G|Zak6r06)@u!FO#eEiuWaY@ z(+(TTo9=!(Y1{N>YI;HE-pB)o*9fgY=XXH6ejfK-uf=gwBhC9#KQvEP`grD6uwhD@ zz4NvbuN@oj-Hr*$WKUaWnlFFBGr97hY}uu^tlRTHKAyT=-txs)lO4Mk#_VhFK49hD z9v<4a*ebJS_2m<8i}!qTSSfS*lHbqWHT|`JZ)oagrfgvBn!eztW=j0C+s_Z^p1=M- zJa?uDiHccO-t$0>5zWMkgKQ5LT|iRyQBDQH`j9Gz@4iL@=S{6 zRE58Kul9$jlr4NuowU2ERGI1YW0LnnE#_{mUwzLyTfV&YY~G=m9j|P~?-X3~OG}+< z5>&3Rxv127Tcz8ghp)#Bsl1^RAz z>)W$<;qH&dH=n1U|GMSNrv8Y&pP#sIWuMoKt8nh{_Wv8J1+?K zS!-L{EY59`aW5_O3zs&EJ{Kc9|Fz&PqbIL^G`T(dRaE+Csl$zqTUYX3pZsLm*U%8n z|1O(j#=)%6H2KPPeevhjAM&PcaQb%cR-jMpr9Dri)=k)Zn7?af@rt|qPNg2xmdSr_ zop9i*jpU|7Dv=IZ%ftl=H01PS67Sc}*`YaOhP&myOXszjI3CXNRM&nbs`)%;S#8pc zrrPfl9A+%9U=!Wba4mNE?>)?M22U)E=J?NFJH`9sIm4IiLB{ud;O*VVV1z1=rv*GyB{>Zk+I*joxAi`^2e<_ z6$+;|^}HcBA3>m#q;wQY&uvbQJtp_*wd>#VJSbnlnXfZrM2B zO*_7dcTS!`pI2G?%X02@y>6Bn@5N&7x=0-C+%3Ry*F|e#&}ka96T@%hIc#PDff>oswmDZd|nf zTGBU((;VJcZx_E?ea7|6o6oEg6&=t;M4bS z^j$e0YE4W(Eh+c-P-P2$Z&TcnRmUn1XT*o~Yd$|)wduIy|Bb1&)(-FXXsx+@o^iYU zXZieXb2t4@Vpy~{$}#`3NA}XYYZtr?K9M`sxx8G&VZv3eTMyf(C~kVZSX*|>gWi=p z9#1eb`DIh?eMt7j)rb#F)$TWsty)^0{ai2Y(dKnMIx|ewJ9Wb5O_p(0w)oYWd}GE$ z%lWRxi@R3e$zAYPT=c4_wz`g0_b)TU;%6XY+ST`A7r)KBUbww$TfF)K<1nFxd&B*O zjeoN)-oHP}=JxBXEzegU-CX;h;e3SdytmejFTSQ`+>m(MV|L-@Z7v0iMG6z2F-Bf1 zQj2X8)V=Cf5+3@y_1w1dMW(Y>tADpXn0wBvz@Ke5$&AFzZIG&L207)t<0^JHvWrL#9ygVT})YPIEKT^;*i!AKDrAb3eT6d*Za1snNsS zJN}ViG8MowU zkLOVx!qr+;*&<-@aXa?ikniylv;LrL{}L%4H<96?c_0y_l97f7832 z_w>p|-R#oD{(02EjT5*wDjw{dCnm{ zZwkxaPpgyGx~;S(<=C<(@lVc*ZCv2@=Cma~_-3(!ZRZ(L>4$$?GS7%JMlJ}xuwvyrgWle0|JpKh%D5t*&(VCf zp*dFEM9Rl6cx}Kr<35&?+m{^+JpJnFDX*>NO}90#f8L}mRiWSZKchgyy-)cm^Ly6q zonfB}8y@9H&CXuWBfa}C??E2nuRpis-OE^#H9zmd4TFyp{W+d3oOt~u=Ug7)%|Eup z+{=hNGx6Zn2XhM<+!GCUD$i%zYAau?`6061ukO+w(SwnNg6VTEuFLw+$Fnf7=uOIu zAAxy0&uo;LEuV5}T}0nm{+wRL%#a4v7}3->dmkw9h$qUZg-v}cXnD@t_Kd>uO2*sM z4frIrZnrlZ9GHB&$W0*aSa8cK$*|p72DhRQCRp^@=bBuyW0L55>@VHR@|^c*-@C-6 zHv=orf96ZRsl)ty*)-L|Jd8TuKR-KBnDn2cYkJT!I?r)32LR5CibMg zj9s-;CjPN{WjU3r@=9p^ow`{8JJP-y7KPpWef9W>d!N)FJ@iOFv-FO=Wb(Rd+lE7l6~t!f~v1;PiTpGaJ25;%?l4?469=`bY|Y@*s#8;)lRZo+j+rW zCiA;n-}6Qb+Fw*yq?j?K=0VsDr?V5P8C4QH-vxKH1Ydh2Tc*aOd~|XG_fIY1kPXjha@*IAX1FS1M`b{=5~kkYF+7Oa`~ zL`v_s7|d48RcpB!n_#SXM$AU$Ys9n0n>?D!sw%&lOQkv;5P5ih{aOA+axqc0Gh{g| zULB14o2w@(sW>m%`dFaoo1155=vi+t+9AK{Fy8_znJ48p9QQv?b-eM&apK~_b4!h@ zUWfLlNxM&d{WI%pnlsCiQ!I{)C&tD1-p!fXxaxQCg$!<`xOM+?WilO?EP8gywDwZ2 z?2K%G)wq52)%$}KC5w0tMuzWw%+pu%fIHP<@?m}-jy7YpE4gXpT9fV zKTW>k^Ew4-WnH&&i42V%xuP44V#`Cmi=O^aT=cGWv*>{s4ymo84w;2@$J~^a#U7S5 z=9`?lvLp6^wEd-&h3i@8{&;k9`GkKHcN#DItFqu+ZEE-~W4;MyZ|>D))Sa<7o3HfU zettcdk*$sildh`Pt4_gbk_#?yBwuB&@VIiFv(`S+O}h7w%!bFTyPi8Qwqt*)XZ85P zg5Avbdvo=^YO+MB%=zQ-T<%xr#<@Q)y!d9kr0Hz2%+2KjXP-~EPMPg(WfmB6=P)=S44U{`6Mse8y=>KLfvS+o#Vn>N-46=k*TT1184}JnM8OO3nL_xUHONZ;Xe% zZFz`jrQ6Qc$s(;h``%wSI~6^{X8xzkCm!6cx_2y4@k3)AgZgnTmV5f_t8B%+4ivU- z&XxEnqjb~CE&uq=H=iu>rv;q3T6%8lo;PcLhn=zc&r`a0<=dEV?QXVWhVi)z&7`Jo zC~TfCC-wBDn~KAQ7a0cwieFACSE=Ku3)#Z(jMJt}y4S0cUE}qVi<<8lET%qNH`ne~ zk>1=-d=0|!tJ~d{?okeveb%WwUr_JD#bf=i)|={Gn5aCTv${cEt@!PllyLydm)zw_#-`van+0xH>OK~PN?VoL|DKjn6ciQL2>#`ZYt}a+&EN&|mIWH}A zfqlHIM6TiDAOExty|{2-w(RWR3!khh)BBTtKjlg88m6XgH33z|orNL>tEa4PSn&75 zq7DB}Z7X}W>ZS6&e5n#wH}>?P+tH1`=5W_N?qKhHfA~LJ(WC0M>*Tl`u6ixokRLwR zV~T)z0OP6y{(f@jJ!<-2?rHSg^;`Mu)n6ZeWD2h2Jh=C*L+r$fHFD4Qdr3v)oxFR= zP3%F(bM0kWKa`o5ESo%k`kNmSRbTrQ53jRmJYi?wJ8xQ8?cC{)6+d6841KZm{>>NX zESj&yEvjafKIgB0{8s&&U#9o_*5-SBecw2dtMrGIYP^(W&^?}&d9r@zK13XI*>3i6 zCHJSZ9g1<%y-gvC>w^5|O=T2W9~CRd zD=8DxetM#!NW{9sTQ7R#UlI>`dV-y6jf{I`*Sjs>decP(GVRZOy~20Rk6pv+3QidZclfw%U99Y;WkGE{rkiDn)iyXyuVTW z_YZ~Cy@fGHXJ+g0TsG^L;}?H= ztv@Bje^Xbijo$sr)-9`2V0uWa;ky&vwG|gi6I45IPoLr}tGSzK_ zZn>iI(7jyoU+>+~dooM^%f#A%i(xq(g(XMVmzhr5+OO#Dob_!@%0A)>9<*3U)D#Zpz{pgU*5 z{Ac@$a<=q5n!7%Av&;9H&1W~w^1k_~@{M5is^fASE!ljQ@9vk$D&1OfPp9|ZuUX2^ z4A(wBH>E7>oW{S;UN6cRO~Qr4LslKoIH8hZV)pu6(>DHNMyAic9J%9Red6ODk^Hwz z9(&8<4H$Mj^EvsneI9d=!Z~rD)W!?DSbE(GdiyI3OSUsuC0w?b>NVl`weP35`qNFl zFY0_6_Hb#=Px`lfzE1DAO-}bO%~+=P*EP=8r0ml%vpybShuKeBy4u~^lk=WjHJiRA z>r?pV?Uj1*Xa6k>JSkm%^nUNNrQA`^pYW!~a@;IPYTK;0i!B<$0@t&Low4AWz8ziu)$V(s1LwrHMrY4D72ZG&{Z z5HBHeKqqY z^xBGdZ7G!cVb-VB$#Z%6yiXGqb@j^cbgLf@eOFP|FTZZf(UStVEjDF|u8GVm+VF8r zLR@mG`iAXEp)T{TE_YnG_Q^RGR`c#~AGW%H^$WH>yjwQIZnf9O+R4*j3eWzowBe=k zx$wv8>7jMCEB5+xyj{BNU;LNX|Cw3KUmIiYmI7TXNW^X_3o~=nJ4LtJzFn2OGxz^< z&2!u(=|69`1$kWZsoT!;h0#k`W66ate-}>LJ5ASn+BeJ8>D-59<8vb?uYdSp^%B17|4-#>G$bG176141x_y1` z{y%@$@Bj1e{PpYc|Jg6FOn)6-`tx+Grt+%&C3h?))$E-&Q90s+<+p>=)_=Xstu9sH z*!l0nZT^I`)- zV~Oc;hRYoig^oYIti5H!L(i!PSq=!FGP^1->9Hs7<*HKW$re79Qy-a|Brmi7yX?e) zrM!3aoKq(yv4LUS@KxKilR4%{__{|OqKXKVgD_OQ+59~@G~0O zZTTOt$i3@#NB4RD-vSFBIj2iWeB=3KYk1_iK%e1G{;!+`FAfMwC)rPVG4YRG(Sr>6 zpo|AMoUM&ijxqEZ@8s>}On7lXQ2tiE#>Mhag-;F$N?Tn%urdDUi3t+S7U!OrnW!}^ zWOlE~_`lHQ{>kGLncZ_<_%I)ycPTMjPQu-S{nRuATz2feGfVxLVBgv&b|z{clJez# zq}fk-Vf{1i%~|IAT)aHhE!oeG2*^8Id=J$-^u+Fp`hkVa?l~QQ0~Wb|`hB8tajnO~ z#^mqUCn_4U738RXUuek4BWv-<`8U7nFF(6J8H*&aPXkU*RJ3a==s8)t!oR_pz3+y= z?}aY+^UGu`3%2lUniRazl#>)n*5p##j@1B*viE&FordpOCf~j=gQ1O2=HtS@?LxKc z^6q>x8y6b$X1|DkaQ@;MeYxIhllsfyz43N-{}XQf-@dipOy_Q^QRn5Hr*2z5>)+^3 zY}UMUVPD+NV>ySnTz^`;*kHTp+FN_{a+c^W4xifR{q(Dc`Z9ixqFta1mVN`qH^{hURKdM>+RGP z2Yy-4c*Y{9IFGH`v^7_C()~oGcRky*=N)rwSSxd8YR7^nHf&r+1XUCcRIa{w>-5FL zUPo3(2=#0*Tqos|<UjHW}-U5|E9jVV=+^e)8}hdssTD{ zcV(;%KlN0!OkiUS>oTU8XGM*1g)(jW;k#Mn4u*BFf0es;lH>XvIj^SHFoq{y*JfhV zQ@znSO|4~Xi}fX&Xvv-)hN{C+pJfu%%4fcwSy(@Hmv-8=d6H`;#(dk!Vk_v;Huc7b z7#=n6s+I}G&DKTf-P{6KygbZIMR#PqEOA}+E!a*j>~!6(vw0Kx{;#+j+a>j&EMui$ zL%gL!jn8@3!_L>&9WuM;Six{7{l=o$LVt<8MNP7N-@?OpaJSw5&S>r$ekWUR+XD$z z{*}2k>dR)xL>Ny!$T6Q~O5grnpDsNOPcW6}S+Mg!%+8x8&DTDja}7Qtnqr}|ob_`W z_pWWar@z=R^z?(eZ^h-pfAb}VdDsmE?+mTSvZUon`iS$!{3+HQg1X}(YLEy`C9%uKInnH+K? z=F;xMB)x|d8j~Itb6#g%GxPNEjB~7AM@_fOJCsdrVAP)4|J=Y=i}#@#<5}JC$#NTJ zye_!?Wvj-4@C2_O&BF|0#TpmyNM#u8mhmZ)s!CZn>xoIh!5&JH%u@|ixv`^O?qpW^A4 z8jC}(AC2MT{?6;HQh$l7^WCLIWyZhU6I^Ef?fj=JQFys(&()6$xNY3NahKvk6&w0L%vrN2Dt&bHP)(v78T${BnW^YW=V({tynOzwoENv!Z9cDx>d{> z-uE*@);?uA$}fJFPq`^t!c_liq)|)opRBs;%aj<{q93nXnp;t^!{|WNk}VGf&3VIA zwr+YaUl;!P-^)hZX1N>os__RWDc;L^cgug%j`PL)n5$d#_0kQ>rP?2wSFzr!{&4<) zF>?z`8@tZIX_AZvwZ509zGs-RZy9H7PP$$u6aQ{Q3Bl{1dlx@kJ@vxjeYehCRs8aN zk@5SRA`kaHtFU;^nVixyFDKqgbFSGWzIlHo|ICeS?*1KknoZ<>WxVj|w1XdZ9N(0q z*3)5pU}l+2TeOE_v)=Is>l@XNS=)Ohxa6>8y!*(-bol2fg{5-q6%5`d3I5)4fBTkW zMYR`h%rdDcbGUFrYNgAj7$4mxsr|tXY&;e$JCokDc09jcE*AcCVTD|G_KD|*-L9*s z7}#FsT@})`we?Wx-bq{Q54AqywiZ7W(q#Q7>*^d1-=&EkUl&(yiGO4#UcSA$BO!i6 zu+r9LwZh)fFFBPZOp-eko+n+p<~vo_pQW_h!NyO`^4Zu+W? za!MZ(g9D8eyVb%Cu14%Ru;j$akYvH_Y|Rr_hM4R;v`^&DL$<4@p9`f4thiSen*VWM z(aK*{M_}X{6&#@bC&F2DtBI~&RELH@=~T={wuRU#IGx$Ia5&i%+zR=jSX|e9Aw|Z8D28pK0Q`!WF6#wtPp$ zANtC?Qg7P1c9G)D+ZtZ)jNL9y;=67Az<*!Nx0|=DA873}YJWK0>e#;ZwZi z7hFj%?caVV{-hv-&Cb+j!N3Dg*Xl^@6P>nwk)7q6##laOEoVc)oB0y5{ypb)uWb3W zrf!zgJr)hWJ64*#qWc6SO#bfXTzq;%LfMAd>b~0gO_dK;q~z;`+573u_!Jo!U8@^* zWm{%wuu9hIbFPcFy^2b>@-4~WQACoK(Xv|I9~+N!CG@q+d+Xel7EGG@eO~vOeKYnv zK6^5uf@SY>rNgP~s*lWiAn1B6Q!nAx;iefT;yov-^50&KFjSI0yIDqh#oA}OMM5u{ zWo}%|lwosTnRi?2-F!=)ut2k5n+=oW^`7|JIEJ)O6JN1_*Z;x7SRMf*n@hfUKz->cTp?&P42-C!yi;C4=FNjQ9eLL%8!sS&ru9!NlGQPBCp3It8Ii@DB zS=8>>%J7RPI;ZQ-Sg!Q=+ex$clH2>&&*u5b{Pm9ad#6nq^GfIb%8M(V%hGl|u#GJ^ zG`L0bb*;OPS7?dk-w?jyo6~FjmVeH=&BFO^R=L^1ri-~uOvT$c#TxR2btDSelbvQ9 z{;-?NIpA)d!G{H+=MsO!-qX!_ZzfSU-G6q&fsS&ye&2)-ePZ84x9>bBw(az?2^SB& zn;771mys0zJ4#kJXZyy|Z5zsGOm2)T;8anazKdkx>5HIlVTL z#p-p2U<{AjohKLXUu)m@N{~5J&m#MK$?ApAr+$syp~J*<`155+-6<)jH`x5lbhcyF zopQ~mY6IVbXfd{UUU^@aU}cvr8n^q};zPynUHZ)^QfMyq(Vw!cqQe6)=u5-U^pSl#4&Vv@e+)1;kCIJti*TZeK9tijWhGdAb)>h^NiS+?nxW<4n28W$FsQm zW^=cl+~JO$C6_Ip%6EO}oXRg*x2@^GgvUC!ubg+S-*7e|MUc^MOWGzoQNb_l&Q1m_ z%T7E#d|zO}%yS=3e25c0?#_1laBRb3`JV!JV;hpsw7Qnso>(h?Sd>HE;LfwN=Rc=2 z^M`R1%(yJwRJ&X5zCerAnWu-oGk&#to1E0Ncx}|$yTXS2XB=F%s-C%`7rfq$Uo8K_ zvbbJGKHtt|ajx8VVvk-~eRo&i*%*$Rth*LG++wahs|+M0Cm)_tvtLHs_RgcDGxB<+ zzhS_%&8mmva!)3-yzF^8P1VX< zQRV&7m_u_|-|HN|rE~9h@auX1u2;sZGNt&s9(k)Nc;UM3!`1HdzwdMj;agm7I-iLGVtLN>HJtj0c@!e{pGlFjn``RxvS3mapT`6vK zWJTWHuWZ*JNS1hiJiF&kvE%%NcEKVy@xGN-dh@t~Tg7~~No@$7yVo?T(sjFh`M#&8 zS&Nt6b$cp$oX0G z>-XkVv}9jYzIwMKqGCt3p?s&bN%E3i;wlq%R%|?9qvH2rkMVRhC>Pho)=zYqna<-o4-OlB`veAwy?-^e1YFK^X1IKoj)%V(^SDqK$;5@B% zy2_d=|KE~NmKAgv&t7<1{cKL#u4S6%bK0u>s;94bxb0=l!+)B;E5-9tX6h~47bj{R z^fcqqL9;-awwEjJsb%db(|W&|eagCn>1!L#RerntGuX6(jk{-|Aj|QYQ5QM=jJGbz zJ2WRP%T-zb3Q~qNZ^X1ICQn%ur zA4KQXeq9{Qbk2KgO2mcuyW6k7zrRyudGs4!N4LwTd~4@!bG@upUpw1Wz*pPzYegWx z`;wKZTc%gfnYt{D{qbidv*|0m53u|*STZ}VHTHepLYDp&Zx0+Ox?LfDFz${1CFv~^ z#jblkEIwJat9nPr)Bth4FSq--U+sO}$@biRbMj5|v&wVssW(16ulwwBAFJ~1e%ste zPPvH+-HZLT9F=9f>%V)dX8g#$S8}&Dh*R!e&7!#1v3wjn>9YE(+Co!O^e)!&eBo%X z6fP(zP5yY>@6Fy<^FJqx^H$BvJt#AO){E5}=FJkA@!rPt=fspe@o!>h(=VsH*sS8; zozQ2eGf^(w=krWX$Ah&IUoyJ-*=N3~%X{DDbVBrKw!gXXrkkz&-81(*;+LMO;gvIU z&tVVO%DZy&9gW2fUQJp#H;(zZi-vUV!o$;7TZ&#gP*|(@>WSQ6|7TVfM`Hg=&q)sM zp66_SV&#kXty!4|M5%^mue3#)fM7QCs~KGVH> z^}NHcP5oRI2=k^I<>6nMrT^XzI%$vXaR@?E< zJFCQH@@IvuEI)Yj>ZXD-8~>brzv|_LD(SkN`(8-K&U|i;ib_E*C#h`aN#C z-@BQ&V=a^{ud_()n`-#+*`;-wt{0=ioSFmq}Ji`&{G%3oGYtzhYo>U2-Ik+jb4I7{34R>}Wzerg7HtebYH)l6(W zP;j0-C;WZ3QKHm|6-e>@X32_U2N`+H*5@n9LM? zvac|+PHEclr7s2F_Wv^WHrr)yoOtfv8KJrF&g+QmvOmmP^eO$z&T00G^B&HdyxPp; zzs~7%(+rNKn=3ZC@9C(YT$h&fqctnOoax{GV$JtFr`EZToa6Ggyf;1Z@@~s1Pe!3| zmd}3oLQd?R`@tYTbgj|64`RWJ<>^b_EO+|Gcxcn%P5J3*dEbw9^Ym9-G2YGk^7Osg z#zi*ApY_~}EMD8^w5uk}U2$&tnbWM4r;FD2rUJ+WKpr|4Toumu4q&tW;L-il|dh5OHMwp$Sw`EO-RW;^UQ+aO&!w{Y$4jGEv*8!y%=%{=ZVz3AtL zncE`LrR!?b@(&hBum0X9VmX% znLh7PGJMe)D5H1&w&(W+8@HE5%U#)b&GX{gM}GuAzx)|9e;uDwwN%mT z;-jxKPxyZL@LOVbWuM8FrpjP%$?uzg7e^;}aT=FKT2} z7Aao8(&7J<2ZC##e=tj{sQAISfAv(u5AF#r9t@AI>*jaItG)awB79KUuJ%jFO(7el zpD*5L#Wp>^tGvG|@I(H6rsS*k<(xV~U%V3z%IGsHNqiL&;LeL@vKKm2%gJ-H<{O{F zrNHX{dJ>iFHm0ka zn)u;$)h+wwUVD6&68ZkGT~GFcMEw6k`zyNt!{1)iZ(dUJY3GAu6E2iKdsY#7ernuW zp80;dHyMvt*93PS;P`(kxv_kKfsqpbHdh^K+cOP^O#`y#r#vsT6QJN8k5Vh z4NcomQ^Ebs99{8+h;+W8K^k>M~NCW??cbY1vcN8Wnzl|1) z%TKg9B7AYy&z6bX4fT8X6i8QcyGc|=L^%Gi`%?M9s4oA>&iSX7g>EX}px^MnXXJ0 zIl!7+xaW&u*rF-cwb=)*sC+na#CNx~mw?Tk1xMyyEPHy`x*_)8xsxw~6OFgcn=F_< zVMYaH9QFp+q7q}L=?dG@-i23G0C zU!$~}*D{$*Px>5s>F-_LruOJ3rg!e2zI0yfTV!!we5qjGA^(qG`Om-JqkcoO_i)Fw zmD3LtuTnhy_jjz-`x)CLU&=duet0-qrvJ~XX-guPbZl5{eEe{*i{ah-!p0fLrqBBO zSoBv_V*Ry`AGvDxZm`e3;h3_2TiNz>(`l<8R7bKMp1#_x(S1W}D$~7d+A9|_=e-hr z!z_4cUAT&ml%#g7@UwgGZreof6z6K0-06^T=`N#lx(xTVqLsUo#O5kpxP1S)(3b>L z$NIy5A3nHk@VumKy)jXwI=!2F!-Ll`vu9m;XCC?J*X7?k-}@;_1k+q(1)GlUmD zH4>TjQ?;LE`}Bvip1M1kIi4#|2(UMkIPuiosgBp~#N*>{FTapKAO74nwtTwTMnw)D zjt%C)W*L4j*j+xaNG_cA+@L)n|J8h7are15p8fmB!MyP2hsZl01=*U{KYRCn$Hbj3 zg<-q*uU&Sz`QW{exu#5qWf=ojhI<~33g-Fd*5)2!)p%-$Z~k1llDD_knfpz=^RdNo zI)B}XT$@CNiX+Z~_4TX5;=j*WfBeg}=ZE^F`sV6?jEv!5W^TRw&X;x9V|iUa6ii>_ zR~#{2@m9j&KOs~3XMI@Azo~Ee><_JTW#i_ynlkLQZarrm7Rt(3ykeDlv9p%v1n zI&XJ^vqaE)=W3q)%8#8p)5UKH1lYK3lc;k)+J4TY;7!~Uy>mU)AXUjs=ZnSN_t~D0 z-LTMa-}9Z_%J5|l9IUOK;fW~-7ymYCNX;lkwiI#UcUpK*G* zJUewE&y1O^Lh2JLR``o`JqX+!aLntB&J;%D%>iu{X<}UuE^Z2FNipSK>TkL^;1Q3| z|1T$=~z*w@XqNemT2&Qvln%nXN`QW<_c=ubtUy^kY_}Mw9zl zr)L>eax{xO;P0TB%zD(?uJKL2FdpG`n#P0jP!g6i> z+{66++n#9joN#*R?z%Bq=BTnyzWJ{Qff670EaWxrJ@i56%ICV?OP|fSr)vJ#?)yXW zDSy<57LKb;>IUlE^SazNKX__1L+lT~tFg`x#o7H*A4066#7^Bx!$#d-ny4IMi6F+J(7YdBrWeZTbo8ju(y9`+ut#EUJ9_xm1+v zaPOfD$Hc$BhTjuNSNq%~lj~Z|6W<5CzZNp>&neTWm`Ic4#diw5UzuDtkUAHU zbc{bNaZ2~0JsYoWkkw@TzfaM6+XHvqo1U$SC;et@R$cv0HtFQ2(u(Ah|2)jLt6H~w zP7E&Fcy7O2rx>?=)Bd6vI(-V$6LKGWt}gVc-XV2cN@CHmZyf8Y(sO?%et75plmC#0 zoAQ*0mRsIGE#`DSqi#4o=VSEIm`&rsOxr++0`B zJg>Sx?XkPK{>hy`dA_IbdS~W(;KJVpvO!n3b^Gs|cId+LP~ z_bXrI{PBd<($hfi1ZVL+h1Fqe-1I#3_Lz%5>1Qd9pD;uB$D6k+PN&`N>o)vo`#ihm z{LFV;g5Zcwx7qCsIkENGJc-_2-XC9QH@*!jbUiTZ`=iQiJ)f(Oz8}_* z`*(J80PDQq)yMyxwLN9M-^oz*`QDRZMTJa4 z%;#3N<#DT9y4Gf*!QAx|-oFlvjsdC(U?;qx^GQk(Op8wDEJ(52NbFqk_fr5bo z5vxVaElrKM?CcbZONvqxbGa(!tc`tpDs^}6Kl|{H_Z-6h{AFA^p}0zIe>C40CNE)* zB^SQ@TR7?M)*SC?-*(?VJ7My{#k$_xbnG{!=>JZAa9&>Lk9zp`uk!W(Ki;?Bpc;8y z>fgut_W%CA-oN4Fsf`!=_WwC}J7mJWx`*@Ye}4}Df7<@vugCrMX7TlZLJHW!v` z`aV_qZ2o@kOScokiyoU7{rG-j>e;?P>lsS~css;}&T;W2A z`yAV@`%!#SZ+)A1rc{4$Z;g^}cfO+kxaY1|Iqx$Vv}U^WUd|4uDi|9!;3T0UH|=H}HR#kixpdyh?gSa>l< zIr*i%xLI(@;wM3>|DT<(nEZa?$A7|)gUmQ>45h>GrcS=?s~t3}=H`u2Z^;6V7l)7JA*8k5)FEsQ!OYPDu z{$(339ANmhRpRo ziN0kuh@AUf;PKQm?`AMXHYomF-4?K=l3((dlF6F&JhhRV_MN!Jc{G07g@D=oqQAOQ zc>g(N9tg`0dTuhSC^G*0PK90WAEd9ze_!m$T*u51d+C`;(|wI?BEH`_>JA4>-4f4H z(TpthOK#W~<9SP}XT$6pIjdG#pV{rZ>UXEXTZ5kEzL^UdCl>2y%@@*3?hb8dUuG*L zb&P%4UKOUwx>vJPjNa!>VqV4=QYe2cbHnp_Q@3B+#+xVfX9>$gH;Kt@Oa=u>W^YxF z?9=(NJY!RQX2l7<*jBbWb>5@R`wq3F?f&Xon95MyUwYYkQfZFzqf>L+WQ`liT;FHYj1C<5kFie{70vwwk5#fjdL;E z#zil66sGwd<`(#U?t+Pcb=$LMjWR9Yp3{mE3$x#=FFl#_Ht|$Po+lUTIm}7Hr?8!TEn}wskMPNkC`9XiZ|T~XQ^Cu z_j=q?lg{6=U8{p1{8I{&ny_=(qEnL(ngm%${&8Zis(iG+L;lNy;2OSFnIQr<<-||3 z&Gi20Dy%NiFs-55Od zaLhSl#Uj{6U2h3H>L~I>i1VvkX zyzoJ|-bJzUd9lUE?K{(^^0yqnxU;uqUX0?Yx7yj40!)~1_Zpvl?N^wgl)R|@Wc#~c z9L2x;?0yC;nZ)=}Kx$bS%?8PHuP{WO!Qz)&~vB?O}oLOe}B$9m4&lSS@zwW znp+XaD4K50yPrGc((=Gl%8gAya|Ud_idTH2hu&&aa2 z8lK#+Pol?jLG`__RtMRh`DpYUT)a)L%V#f$VfIMlhD6u|%Pq<5k@GrrU6yZ2=+-p0 zoyNHQNzA5Od)RI2{8TeOu|zYPA6ay=`FP>giuEt$F7Um+C+;ks(Jm|6aaw6X?_2E& z4<$@C&TaWR<&9{}olFhs4fDJ*W<2|8DI=n5`sB#3B#UJS)-0bBa#r!%#%1qjAM*EN zR-W_5S*~$joubdi zRfdO|_G&FmY|p5O6Zg8hOk0g5Bdj#ZaJKf*O`UJFmLB}<{Mz($XN8~i_oL^|*?gR{ zZPHP{p9;5LrcZqoqVibJKcg*qBLCMlwmgT;AC#>8Q7`X$WrM}JZ5vhM)p?z-_<#Dk zFmTfPDg8AMo@~$C-~IU4hZ?a{woV!W?-p7*@3b^AnzH!tccxQ&jZJt@^)CFWFl#b< znw*92T8$-)?XNg_?R{5;pE~=p%k9;{g?*D_tW>z?6qg?hHVJxvY3|24CL-&86r}%u zy3fYRJv!tr^B33574Ho1+&trXc+;YPcF!hcDwxR}mnr7STxR4PzpnDx$#$pXO*2o; zZF?Cf^uGJxhTk5pUo_vgET5m)arx?mD0f>U`Rc`Tfsfd(ROto3<7$+esng~1L12Y? z){}KMO0(vK+A*@FFJI6h@GYC+`R^YajTMfBTgyv5G|_o3tvBiC1I1*u(=#5e?%}++ z;Jov}mK9lzyr-tly27Z#pu5=4>OqCp-OPw6(PyjLYA+g0VcR{+KQekj$CIpeS>bxj zMjPTjvkGnwozk{Q`-A3Ik*h4qX{+~aZGR|OK1ss=^iNZ+LbIA3OgdHSJfB$v8V0t99<&du#3( zIDc(_(jv{~#a9CwIzQ+n%+{;=!v62%cIMPpwm-pIrLSLyF1xez$FmO#22zJVTgI5E zemL^-T#(_-GPjj8ax6B7rD*J?!und~W$DC?tQT;ihvhGm{UygnfpIIk<63q!b z%4t3!xJ|&{BfhkCM@PxgzJmfylBI6U5pB7p*z=r; zlxXIrFu^+s>pg;hEQk$|+T^m>{YA*!Z7$aHEZ@~y&nvXcR+iYC8obss4+U+SynV{UP{aIp?)UGM_kCZQ^z3BhOskvA&&`%uA9T=F-X6K3CfvT(K7=Zyl(-qzGzG~Kcz)%Vl;I|sblE3>ZaOnScE zHt5-Q+X;PlzTKF8%eVVj?*3aBXG~N(pI80HH#Vv4Hgn{~Wt(~K?|b{O(Eq2#HUF10 zvkY79GoOh3JLK;3P`l=x$Ui5gWgo<)3|seiC@=dUZkNPW|4@0^2hpwbME*G`FZ(ck z-w~00I-e&WiuX7o^3Un>T#epZ^_40@41;Z zH>=X5=4a|nSH4zm`|v+|RQiFx?kX=6Hb1)Z(DMJn?d}Kl41-NIZ+zpuSjjiz*9;RL zKZcBTY{x?$GG7a^eYcxizH6I;(We!+PPNYsTDE$NM$W^-?^KVRex$kS*zzUZPR~}n zYF%xAG?%5lvitGdGiE&Z8GrqpP#V7Cbnqs(t8Er5@AJ+zd82TnibLw7Ny);*jUiIM zA1+BW6ljR^&seef5Odxw@&2WkEat{OSr{yO$YO46hK4-H9`1#SX@U#&r6;YbUb9C3 zm0ZrL$2Elu7dFV>@zDFp7cze{=*i#b5lO;vEIV#irMa1PrQcltBR z$TivZCzebzoO)dHW4_+Gl{E?X4(hx3>f1ctUiLltPgf6nWtmjml?m>tvz3fluKZO9 z+Qut?*1}=RaY>&783nCc)pF6vB{L72W%0e5Q*~n(11n1@&%ykP)xTNz>y(ey>J}Hg zc3TwwO!z=P!@s-FyBnilbYC)fFzev?<9DBFzA0Kztr51FcTKiNOI!R`wwbGM9-9#R zl~0sOZ27NrmvxedUi9+se)U!%;pTPm-LK{@(04k#zUsI4_Gb)<{wII;oj%0FV0!M_ znm)dUS>MeCJKIMUbQ^ZtY29gd^gZe z?ibhZw$o2OUJy?*PHk@P+?BPve8-2My%!oTbF{RjGcC~H#4hI*QhevSp?+X%Y4F$5 zo=%4LgRR%Ze~5DY+LUG+v1nHO85TSHr0;L1but$m5nY^jn=?PJZPB4MEP1NooB8=) zYCk;d`+nO#otsR;*KX~s&0ZchI#!@pzRmiLo< z=5V;p*qoKXd7(o!=Y@v)g;gn65329UfBA*iQuIRCg$r9*Blyf1AMu}f8NN#HVBLjl z@yE2zOxPy4ML+3%j^FQfQmzN%YORH&R?MLHjnwrj>K(Q)gkRiWa5n_iT(*sZMd|b%q)1Y@z%n{MokYkIPET8vm(y$@r={Y^fOi(DWxQ@ zQR_38_-I+m5#2Ji^ShFnQJGWlWpd4SH@H%-_VWx#HMk;{z_)S0%qDNc@%RGm6fTRINNaBfVy6j{?KO zsL8uLnj2>vf6qNZ_FA-r-227LsvRWuN+&L1y(SSSXtw; z!iNv1Ca&5nen`M|`-Z@YmUk5wDj6Sm(bUhH@u}e_L&Y8Qs4xpYBloN8A2i3T6xe?# z<5R_6PeJx?D&9W=c62sMz3!eC=(vEbV@t-Tgwmb=r>AfJy&<^Ll4FDa^@V$E z*zUF`T83>q(k^@IwC|n2!lv>%$D8I}T3MTTX<>+{{E-Ov30mvRdahJWSk>e2r|c1W zjD4|zQ!&?JpTN0^-G`JMBTQNvay7#q9qIV#5K|fWMMRTxS=pN4n-&79Ah*lETZymp<5tHXQvn?}pO(hnw7%m`v&Sf4pd}68|%$TdKQS+n#Q^vm;~s zg$*{M&!%58YrFFMVJqJ?@u!pjR2&PLXqjx&9==jsYT9z6$42vv&Z+jCt~f1mX-f7= zW%jLKT$Ojta_7r>=*%qC^Qbz(_~(Vw{9?)si(lP)v_|em$F_%mg#Y;5&D`>E{sVUz z`RDT=qm}?Ht9@VhQ z=cnks7X8zECUqurT7OSyY5l4ElxW_LmfCB-y)^pY2}f-Z-L%VZV+eOmN#O584Qnjc zujSseY<9Gi*S7SU!lO$7-dzp)6!KuTj&7~W`6{3CUmh>D?!Ht`onpKo#^`0q z1n!1ATk@N9Y;?-Mc}S+P+->2n-@f!f+Qd+1Z?*neCXruL-adW1Ph-UyhIfg|9J@bW z*eCO>WE0l}p?%J?UfmPt&HJFFx-gNEJ#NVtPtz-6TLSG&ZnvLziCS${aAOL?nfFSU zErcI$-1UB`&Th8fLThA8{rz^@hOYXo`9rgD(JX(hw~4)XYclS1KNAkV@FU@jTddnM zm5SQ3bmvu~{D!wrGQT#Ap0Mmf(%*>P4jV395D`v0dTZ%-&*C#(rE|Vt*l>e?bJAP6 z4^GaH#r_{LbCjw+kiXb3|I!{W^Qb-U2ajB5e;e@VW%j=;{q}0{cfSMI^7fl&NMBMk znq9urW3qRuN#*A|cMh37al8F(zMK`q%xdrK+F4TT_j*ohD&T*%LMWU2i?+{_mE{e- z>Qggrf4l6=?#q~C`NCxuizY|8aKF|@vjvMc?==0&x8dayq1@YbRWBB3Ht4d?^_Z-1 z^qB01OOdLVuDvKT^%O3Gd z`Dp=~5v&jSE*|w%y_UulT%7$%HsP|T>aA^mzg%z(W{BpqnyBX3HP>3KdtS(*To3mv zcVIG-c-8JiD#s5R?N=rkIzgFg+Z@JJwrtmI zY?I%ycjja>)pa&oBOcAM;JBb;DEQBHOGUcy5zE4eJ&S!i*lX_2-}&=jONI1(*@ud3 zee3?75dEHDCiz0xpS%L_FXy1eXr;Bac*AiyLyjnU6(31JpNnjTB)0S zmtijZt9m9KgDi_bhL-i>AAO}}t?+-T_h`1smlqs|Jd>j*MSdxb5OI)YpUj@WMDYLN zd0~y~V($7*5L0(vBEiXPf70u*v&<%o2w{icN$m1s%S@uLeogvVmg6s^TYFK8)nxuR zm5GOrH@|zk>zxvR=d%rJ3-(8DxNxC)p~UfM=1*Ia_uY@YByQX6^m$>;4P(8Rr&{=R z{g1tRn^Qh>rcFlD+eqJ=JR9>8Qm!vM^X0(mSsy1lwED08`&n_5#gBakE9#4KxqYQr zyB;MSU%V!8fks~f6E`z^m`UfxZ4Ezork(m;b|-0SuK}yUkGef7pAQ~?(sk3kyef-L zK`}^bmDP{sw=VN+ULXH;ugcQN(KUa~l=M<+R{UIiVg0T@Lce^aB!B6eyh&Rj{jT!T z24VF-KXO<1&NF$lL_?vkNSd?Sh*cqab#L6RB>!bQKJu!xp8tGyODq3wi7NYwaM#s2Gb4UsU;CeFS5L^jUy(e0((wz+ zZRQ<`{FSuxPN3rc2^;ipww>M{Wazp!$=&du(mC$^&rjc8cFW+)hAaJtGHynE{Mr|O ztV{lVRqC12e*9b)eymHsc#l!onsv3QYa(KQpNe4qVVkw)PxbbhA0NKwZ|ctc z?k{GyM&}xj#EKr{GxjB^rp?Dv^6DQ3ZF`aNXvZ4G*K)c&{0kG;|Fk^(qhN;2vFusg z^547`O6$kynS{u>7`C+4w=M3;*U~r9nLYcMF1Kr%i(%N@t{JZ?+GETNIqZ(BPo28% zg2}husa!FSLTmeOJv!8%z9XjF>ZNJFy53VqzaN*_7CvOyv1y-1Sk{YecMnf^>UplO zT5*%%w|f0p&1{}Y+CBL`p31&tv5Hk z-8sp1-r?qkDg5?Z_S{!m|A^bCqd|D;b637w&vGU1H7v?~W#8fQbo%F0vAr=zmv7k6 zG|}(n>F1wU)rWuO%boRXzN^`ywMv@|kIcJS%=NC$A$H-%@64|@t$n1fSyg%9Q2WC9 z7V`>I&9*5>AJ3h-S0hbA{@;(H2d}#q%Ad}ex8JI4!q0e(Uk{D;cl~nOnlE+X!>qUa zo;^0ZILq#l#p?RE7V=U@3;pc2#2wDR`@uZqeZH~JtjoI36O)bRN4)IT zb8+)p3wAu7&DCwQ=E>bVE6P{-@*A#7tf`3n^Vi{(le2Tygk}mSJZcz62jyXWXXqwE3RZ!Z@-uR|sc1U8d!6zNh zr@;nNCZVBwcTMjW_@=mzBh#Wouv9PXHMjSQ-^IB_@fRNnzu8^9u+H?B`YF-cv=aX* z+P{4EY$@xQflEdOZ>h}1O*S@$cP@TfvhZMaQTY}fe*R~3KOL0TGc3=TUlmZ1 zB4M}xx13oG>u<$ zjJB#*O?>LIb^gV5hn*+Jull>DT(n(jf6OoLyV|^`HIL4led&qjkIKfZ?TKmmcUtN` zmb~pgWccTwPMk#S&wv;0@qvfC=bZ|%kp9iL{6tUxQ;|c@RlgM9HrT`X_tvc+5}$(G zemd55wc4KO=kM?Ob1!7C^8RH%)f?|;eHQu2y=gXKEw*?(l_Q=L~l|A*jTRSoyOlo{=h zrrI6wFPzE$Av#m#!Sa`;4fD-x%pn|JfA+dN;z19Vf85bH->k;`LvYv;=076BH4XDO zU6OyW{L5E{{Zso{{!Cd})o}mTbe2C;K2|o&Pm1IJ5d7>BV_p86Jk7nYA6b7mJUz}X zR$aums#0mk1&tl~mo}K3Z2mo|5@EDcjnB5bWfl9A8wN+xfwqg&MJ+(@chiC z_fL;R#h1&TJk_2y(d*6ABig*HPaG49j}M*sQMou``^Ha+@tb5!m9%+TIm>%H7KLwF zd42JkniuDIDz;rM)6mmOnzg@KzR>SuV8e{$u%9*yT*WN=nj+sV6q9<)V|1PM?!EVuu7nzL3BQ|kWop|a!TIlk zFJ~sV7hNvn^x7EuXI|U;<8wFbpITnqvT51j7kU9nQ%-yNtH% zOL)7%^uz>WkczdZNFz?O)rxO@7Kbl5eN+$1mre(qU`4{>9fROR{S3s;)d` zVXwL}Dqik<`=68BbqbWf@bA%1mJnQ5c0)<;*>086)WhdIUe#Vx>gMqo*_ ztC`VT*!g$%>st(AI;Zc5D_Cn9ba--{&7ZQ&!0u12;&t!nsTVyHB;GFe>{_T6{@Z=B z$m7h4#q-|;*cmC|B1{o#GF_+M$D^+R2HNv=)2|Tm2l|? zmnM~fxgbFUE`6uO;#81~g1%!>W@3)6Q+`g0ZwQyZYhH4GN@iZVg1%=;YFp%F-k%gzquCeX}2SPh6_ zW1|nU73LI>5g-GLG7C!bixdn&E)K%&BDl+)JzYRrK}H#I={tM6gyg$>y7(p*AlwQT z4z4UNNzL`lOUqZVvC$7oP0uVYDXLV^a7@WhO4a1j_b*CGEdqH_1Lj3dQ1}!SU2a1_tI9aCNb z9NZuVN~|E_gA1Hs+%j`gjT8(`z)=bo@k`82MNcelIf?1T3T9mTj-cdGqF`xYs%K88k?KxnV6YaC>R+S8|#@_8k#5= z8km^tS(qA{q4*QzcBnsnQuESFG7J@r&CCtWxb)pJb4pT+6!hJ45=&BDQbDPgOCO>} z!O+~)z!+u0$=kh^HB($KS;XBv|M|>RPa{vYnJ16hOimFx;ppVS;jSX=u}Gq^VAg_( zJSSUto+z<*>}J}eA>i_nL4QhxV$-P!3T@txYxOH0ELyNhYxRmXJ07tnK8wEl*@w@i z{Cn;F`}X_!jjsh?y&HP>YUtIgca5D{m>3v0@UWc7$SE?jli6q^ldwYiKtS~S6~+5r zev)C$=wj@c8}od}wm^A-BB_QMfnMu!SCkxRdZEtHwTxLpe&x#So!jzmPOU%sZDK^; z`jv%NPVqdIyp@4$LK`B!9$U+(JmFruK~mn^N3YH${Vi;W+|MxM(JJZZ*KAe(T;?!X z!sf7ROHpF?pYw;y_pwfBf4tf~?3$(2^LKM9v>kpVHf)G)cZ18n4%YYR|rcTP2+IY2s-XG^NZFQrT9ZvZ3=FDmf_~3grmbGT>E|PmOcG(N%jU?Qg*! z%QM0Id8a>!XxPAbu(4pdiSV`L+p((x4cf0hjxbM_|LAeQ`Rn)9+YaS=-0yvx+A}>_ z&0bjV__vTH?1EI(g$eOHO=e1>2*S zUHj7f>d&S8h`A{fZY=e7Gt29I=NVla*dA(a;h%io!IFF0btTz`s!YMxf=w7QqeQw~Dgty84^> zkwJ?_%XY!MU6x7ImdR?(6>CA>3DIX6eIaf_R{>o(8?tH!)-8F&Nd}L3> zw+eI`M0{n5S$iPiN$R(mhm)qQ6|cXqvn?X#w_*RwpE~bOUeB4Qv+UPH>!o=!4wXE3 zoRss^FuMH4`=qwkf_0bo9#V`ro;2;~Y{XlGf&%LV`b&FU z?Z4e#Bre}jz483aRkscN_BVPyayRg-;8yeO7QEm0`F3xqh5Sc>XB$Ljta;g{_~P_Q zjq3B2Kl0dKIj6|=m9E~wn_I~7?s=K5%i|-?E|a!Z`g=IUP2YA=bw6Q~ z{@(s~``3TY3;auR1A4z)zofWc{CTl;wD7BZ54NT^$0P1HZs1Xjob2@O+8OoOSCR<< z5AU^IP`H)2C8b?vqx7dE1sfL6&55vgF4a80`}KzZ;U~>Ei2W^{Y$fxkRd|MUq0TSH zcl;b{>W>KjK6=ecV;sduLz$ymvjsdarchJ?E>kx7L_VOV)h)kg_NL^RDx?xoiAp zv6d&^S=zn&p)9m* z>(w|vU+x!JdMxJ9@mq}HS#R#VQ@X!?|M9OMd5_)TiBOf?yZ_UV`a`cp&pnX(#dcks zxheOdfMzZ8_lHqYN8dg^oPFb$;J(HJF=dPWFaOwlF9asdhaszsw3}f7kgWmLz}R*s))le`*eW)7<@CAzJ@cp26z37IvQuWPkkRKQ7w0 zcwhMetuv}WXXQUgoOrfR%pAv{7`nuY6ru8 zk-uL*dc5P?yXKmQh8H@oR@D8Fns&J)LeuBa4(rb9wuYhzvEalF1wvh|@jw1;4!Y#X z_q_W+?H_Fo!P5czq*oo`TOqW^W47S6WSd2EuCQ)i;$@Q_+x%CvK*k7S6(b26q`Se%jb*trPoa6s`xzhI5H?|)3W%`%?eofu=?bXT`6^eg<*hKEv zI9}PeVe&5V_quVj`Ky;**t&st;oQ2G=O6MSoXj31rYeg}e9R#&S|OzOxPtwVh=$Pg z385US9E*0a$T`eu`;pz^%pooKP4Q3gnKtQ-@1F~nZ>nZw{-yIw@5qgR8i(py{Qu`5bkMajD)63~OuxNfw(XO|FRr)i*>rERU!RaHQ7`?Fi_L3i)9ufUG2gr9Z>SSpZ-2gCLR-0I zWrF>FxtC9Q{yhEs_t*PVH{O5y_n@EIeueQzohF64rav6drx)7q=xppd8lj~hfBr!H zJK68w6Rc7e&g~Ll?%(+NloswmoXYDs*Jk;Cm>5({`68Lyp9 zZP>=lc4Bvvqy6q_M!O%)@Dcm-@8j{GKmR3Gb?PhL7q3X0#~1kg@x32mpN=-nnE$Bz z`#q!6s>jbieiTShN8|k=9M9iz;KQEnUv)Znw z{ET^`?tz`54C(XPU(D>Sz{O&XNj<2eSg9+EAgZ|z2}mqd`&Q^y60XYy)d?? zEqiKls>(F(+D*^RwEy0$`Q-DCXI|O4z7Hqo$95LTU#fRKpke)6>5;YI|BgR-Q?#A; zu)Ln8eC|*Yr<35c`v*TouylRNXc6XEkKFfY|IPhoBkP`}J$GmQ{P~+=-8_|3>0L6LHyc~;-vlC*?sfcdd&%(X zGEex1YP-wa&hY^XZTcJ+{Ei9*?Ftgus(d4T%bVl?iO%0Qf{L_v&b9Hl^F-t0C8J+j zwR7uq1RMpnIPKZsk|CTYCMWVw>($YdkFGw-`N;QC?_=G^#ErpswCgnHO(^r$_loyk z9$FQ;<6UprnEUM;&Cy~=r2Jo{aX7vYu>eAC%a78WhC z2{|{R{q*9eF_mj#6wFU}KXrd{{E7OSNv3C{5_>nRMYTsQk6yoV-S)iB#f7VOJlJtz z$A=v&EWGbb*fC|t=|?h!=PIP~JGS3C;Feo8`(WCZnYo7BSfW?yo}QMydaLf{wHwxM zx|_56@V5nTIiuTi`*Y{NRlPlX8}l~)-2WbDe7C6gd9U$U`3@hx{&I-X|Rdb-Ne$Lyq0l%AySbj|)C9dmx&?*>tMuNhAJZjMWhy`H^O zU`tHO9fhMsqJ^P%iax!MShwVQ(d(SI+t%)#R?VziE}?6_>w3|-BbD)jd6q?9Hr^MC zPNusrdio~)_(6gW({G=!+Z4f>{pW}8f<(1c-~@L_Ue0YEU$ge zJI{FRX0iYIySDycY&zasS55o2=0)Va$FC&$K2%yBNO!ND|4M$hVB+8V>#epuy0O0g zeo>C#w0VvZ?H9`4{Mx_0&g%W;qsPD1%&U4c&3}8%u7{I(=i7WaDb?U_W_;7%cfQUK z7UMtalJjraOaAjbbaBz*R`=rmJ}aw7FFq`mbZ?jGwU(;AQZYs5=ktmAeKn;|eoQ<+ zRzps2BdVlf0=)b#ud46qv<-4PPZb0wUYhDT=o3?tp@%)+B^zSe`^Qq+@&Mn!~ zqV=PBx`AiJH*4MP3J>ln`3WuW676{YCI4u(!h^rPhu`l#BE^@rmh`c`n?YtvsSb+3`R zKww_e&JX@Oc<;5BKeX++D*Sg7;a`^H@@sq(H7W>KWCl;p|SN7y6_D^1$B3;>J<2mopoTqBm z9_%k$;vWcPDrcYY`($#{b)I6Z$^@2)cT@N|)!2lboE}R2ncE_$eAD~e!~!A5Jhk5^ zH$Snwq zN$DrPpTwS>**~dbl0%NlL>UG4lh#jSKgm_ff199UG=oR+A~ns``oYCo(??|D5^9sZKTBgUiyp>V)MZ#g#ky8C9nQHFkw8(pt4l zYSmJ!Np8$5=CNH;kn-U*X=%G;kfq|BY0<0jQl)v;ikWVgG^Bi!GjwEKbVHVz2W$5# zANEk|RX?1`wDqFPEEo36iY7CcT?j4Nnli!ZMfoA#L-8%rty2$WC9W-4UC_QEZio90 z`SNq_t;bt)4^2K)me~Gb%ZBJ1`ZoG|boS`&>COHs`*6aCg&#J4D1W%4z{x_hM!m*= z|MLHf>@QsY;{Hqc@63Nr^-I%VMBY+<(Zb-^znlA@r>&BmR-Tx=@b-@W?&zbrkAxph z-myAI{O%*Zjm+iG=03XoNbjTMo%FKq>Vo`{?td;zz`fPCqjJX!w!v zqt}mIKWcu&{OI;0+mG(=Vn5RS$okRlN7avhKT`b2{4sxrcdbU9ikx77mwfm9j`>~3 zkL>TB-#NcKzT^1O`i}Un_)hgB>yPFi$v-;zk^Ry4M}i-Hf3*LI{E^F#`kxekYW*?! z$L1g1KUP<`*>GDgXx=tar$VaiV0VS}tCqPRgml`U-+Uf+C?#=KfRc}38i#SKc56`L zECbm(mH10}-z>TBFsm1)+Hq~4Y_ntPlgX`5%dm{IQ?Fo+2zNd7r%lcDpjsHIX zJ00n>*gvEcZ?EXD*t^3sM_m5M++T@`(GrS(6gK4V4Q2SJ-^DraMB}{aWg7woRHsHQ zzRLQ7xw(l~abe=I4;+&%jkK)P$|ksOJ9bq4hR|d-eosG7U+G!%40Vbt!oGFPnYh@k z|K#43oO!nOcYZV-TI8)iyP)!;sIa=P-b|y?TVEuN*X!ucF1>p7>Gd<~3J%OUrIr5m zj@0!%Kev{Ct8ZVsv*+8i=<9dO ze#@4xyI*kq?)N*@_mBOSzj5W?1808wPcNPw-)6I~boPfMll^nc-@kZq=jWGo>AT9_ z-`t)5fAjfL`~Qc!_4occwR-*Df5&F$@BMq7nf>A1^83~QFR<5J|9R-{-dFeMR1!ZGYaoSEc;Si$LY#O`F@&`A&Pt&zd%Cl1)lM&>~$$yK`F6Y)6l|xVz5s z6?}MX!vz87f`A2f4$H5u2%V%6z_haUqpRp9>y3wW)sDrNMM`NOx888*;`&90bdy6b zI>>F0Ie7C)-m%_BUPP9Io#dOnG>xonO#Y3~U9qM9wljq)$SM#T0xt^|Q*$bu#JPxOF zyYlZi`XqCzuUJ{|g^z2NL&zyv(fEKDeLeaKQaf0q7cM>(YqWIB28CN$y58QUlRr-V zSbRso{+#TbLy-wK^2gRny$}1Gm?A!#cg@7|C1u<1-brn+6xIB->W9|xt4s3^1|B=c z;d!u-`_8i0`wYH|Zv1@V<5KP4+ugWxThpeW>B`O3PA&gF<oFQ>@#-iru(rDpzUg3)Zw*;DiD&i!x-3pVsG%YJpRyoVHg)mki+{qji$BcLvp#dMsru%Wp!mo( z%ZsuJ{{EW9m$|N8dYjKV|6tU#pV3uEwI3NyD*xmizax6`joc65anG!^sJ&NTbkRzuiy4jElLF%Io<761v}$|C z;r}5vD^}lOVpCnOP&Dy&K;g{yk;^tm)rT+hUi(u0%2V5;ODtXq2X86q_mhl}GhSfh zU|l=A;a6AZmWp3%wwc{+*rw#;keloJu_!Rwb@t-ZyL0^Z2EoD>V4BASYH3rmp!N$ z^UhyZGSY6*<%DxOC(XI8GYQUdvOUtws-N@KfTK|LQN)qr4-B#=zHTXJlQx?Eu%hzB zJoT2t0y*c7PcBn?#H7fz$TB+HeM0`6Ykdb#&irYWxF_M6&Os-I6vb;Culjtt_k?d) zysRuSbh%l>%k8TU3(VUo&(EJ_pmg2niBV+JcWx^W_XOja_g3U@e9s7^OSuwKf>8DfB9X3_8*o< z(vIBS!n$5l?3c;eHp7l`tsknp`j48_w!Leb*kNtmG=u9MpQ-b0>kuB>tBm`CJa?ta z*LLMKYoyd}Qa0qu-d>`n7SZ%QslLU|E7~eaci~T~UkaM4%O|dsVDseZ%4wS=BchiO+?~ra;&+ zhx$P5;!d6^CmcY;10hL`iH#uQpsG=T60@q&j+P!FBd-%0ip89gswq<@1Ufx~@tM6O z0uD6@vFRl`aDv$fRC(^$ujTN0B;h6F+?lLV*y9bMc~tufL3|$7zKgEbEzLjm7oTtx zvFuP{-wCAypD2if_<>Ir7=0DX;n z%)@u}@I0*0tU3Kr$l}b$uN#^l+aFHr`_FRPavxujMI1|~Ome-<_6ApHD;>*&KLuwn zsVSdv%W$~SvO)HtC5!2DK82dD2TN~$voMUXjojzGs9WpomMuHD>f|clRc}{4 z54lu!)lhWm=f#-{=NHe2dJy(6rtAEBfdb)!>2k|IWr+37Zml$vz8`6FTCOPQpWaLv z54H2Eli!|vaxG=j`lZu_CZ=0S7JXQks<^bQ_EE&?#FjZnmTa6cGwNCNxjzoY0owyy3&HlqdRIk88%hbM|oBpmZXKqa#6ncWYb5 zo86fTH}Cu`QpmVz;n^IRm+v0LwrE}7l9L_tUV6+*K6!mo?1UqIj!!bb+%Y?H+gc|+ z;qzp>FD~t=`ipdBr|GynmGyKMjJ-PX%QjW}31=9(Pq-}Cu8=x6`{7F2X){llEj?_+ z66oa~r1aSDSYyr&tClH;#5;1kLYnfI{NOt+;Zi6&EwCq=+4Iat7uL)eTh7MBg~Fo0 z^2{|wJ8cgXT`FpS8ZGjljko*Z$Ax!Pm8=q8D;dAFI=ycT`vlATmnN=H{(Y3t4xZg+fT~PJ%UBy9NcVA6f?Nu&N zTAJWv{!@$*1qk?D=qPn&|HQgzfk5n#^Dm6MM7Fk!R}EcVg0+1rI~Q z^v%SkDBc#I!@Jm$N=BkwafHaGb1?i@P@-L^Loq0c1%Aae!{wKZ$;xX@yO2` zel*E`NP76?lC1u%xva8>-yG(8EX$fYw{qvwpUlUi?plXszGJmIojP~wrj4II>0g+0 ze%9w35kGU*)R=|Mv+eH>eUVfcAGFJv`SGa_Os5Ogc`qdLgu6taKISSRb$nC9J=O;u> znYK~EvlzSXp1j<3SNX%gz0PZoE!5&VwIOfCem{YVyL@Z6Z8&|qw^wroOJ?F>L#@^& z3s~P;?I~TExqDTt@6FJKOO3V6HYeJ@d*ChjLC#&gbjj!6MzL2U)04kXpLXs`j{LFT znulcruldL9*3$*G0E--)6qG)LL_QtE9H>?3-b?Chm)Dm^LMC zzv6@(P3PprQ8|0qt&@^|9(dd1;i>7JsdguBzp2?r>AlxCob_6NcF%9I{mwOC&pt^0 zX1Z+4uhuA0!FxAOxlWk!&4H)=xPqI{5pI!+{U4f-KmFUZ)zD}EF%#L>%ulZ}`X9J| zB0lgzvBwCkTBy&!9d>pwD^c_qYoOyf_YU-wqLC7QkFTLJ+e7;4- zV$FN}d*(YSO*we@1Iw!^PP>ktKEU<)q1%qxE?nysw#vyd@?X4Ml(^@MOrHs3nv<`w z!3X|}3Tj=>M-v~V6i$2LdOpQ#y4U3;x|hsevc8<66&|`YE2MU5+`71hm9Jwy?ew>K zu=3*$+qn|ErFxdUz97VZz**wE^+opl1+M?Qq~x0~zi8D+QuSQ%T_eb)>!eGO>dsjf zCyO>sy5sjw?fb;`6TueDbC^RHztc3C@3)xm)e`L&Uc08fJMvxbm(||}xn=5s#ghf} zmv9_z;rvrQr)rKf_kY!+xj*h{$q3vN-Y2q8$ZyegpGOKCL-+XYaj)^Yqq$ExR`Ke? z--<6SS4LO4znWMT5+b^^D{$%RXDhd@unS&𝔐AtE;cRU$xJW^5pS!JNxusyvSvb znC9jeVOb*6CryqJ><;frm{ZC1D^Y4)m%M`ihXwIX5!SE7{FILxsIA=8o%gMg7p->nzf*mjpni6#@k=+x z6NPs^^(F6#=PsZ0bBW}(iIY=IcFub;Vd)3EUV&v3MJK$j?dWpxR+|vqAa#H@Ni^p3 z@%t+$2NsCOrxX za?=9Mbu%7IsQ%t~#w+}>*Y!lblSe{hnKm!rT6cU+`Rwa&@4ookUA}QU<2{#~#wt_i zD8AjHv@1YKJnr!KwI38Cn_qvpWIylu`@J@y7kJ}^jW$EYmXi0C6(5-zUi@p>Ejg?5S_hr zbD(2D=bPo8-xQ|K@>=BKu-JU&q6OVnZy3c6?N^C*W4Yui{3fb(&F6)1X;%8DtpNrJz@o>scomZ`=s#?C(#3c3U zSN*jzS!o`*C(g$HyySh@$YY~*Le$3U3sWMO?soi8Wo5D`um5`UvX?g(3b}6zmy8T} z_q*(RmgooHHHl3Mw$CfeZtjfjwq1Y0FeT)Z^!f>%^ZYXX{iUa#mvy$k%c`)>!8?25 z$$85sx=c+8K0S-)Z1F6A)1+N#bK9~^6gtejR*0FrvbmZf8tN;0!(P0560;FgM#GA! zoW4iBR@jStleHb!R{5-Y5fGg5 z<-vr5&khDQvuEmY@9>(lZmy_-|DVYfb9bs{gQ--`Xi&YwQhQA|&o*`UC9K>H_j>2_UiDmPywY_V>ztw( zH<{jUpRDT>6Xv>3UnpD>`{kM6>q)-*BK2ofO!A%Od#~Wv6Q*~eC#=h@mj3CUvS+g5 znd96+MH{Ok*rrVWR5U4owAds+X*^>)`yv%h{$c~^bh^XePURYpSX57=gQr3NGba8ghX>@cH$W4&r zWAk;nRA!p!y(UtvWo|?pa@b6sN%lB^z^K#dD9sK!y=d5&2 zjd+Js7cXwoR8qRLB4}mk$Bz?5o|`FIYa41B*4bJ*H7(P-Y4v=a=jXWke+%DoACE7N zEZe=o zrgc~2q<0#WO)U$3;=55RwM?G9aIa=L-Al!_2;d+YFi~ zhDhA*y^_Il^VFHvHg0xb0dGD}feel%jqlynx30dkD(|Y})D=-TCH$ZK=&6v(=Y5~Q zbw}dV+A!lO`af3wdGhkp}zTBMkhwavfvkO!8ZQ=|6?(sX%EN|kwf0@n{iDR@;V+{H#IKlXL-|i?ZOonVviIj`)s#!$ zrx&Lm_1Pe6SR=JaRxnbeGBv66$(O9@p8WjF=uM9-CU zJQz9sh){>=kzL94y`ss!%6P-X~7i)_RXBORGvRusefP(|RwaI|-;+$8f$d-zG81tw-Wqjk<)gPuQ2xNnWDz(`U>zO>*H|_g!b( z56)<{hsQb|&Wtcsnr*RH^S#`S>!CU>w{7CtC9WJw?Vn=5ss35*_Wd=kp3Wajiq3Yd z7cTPRUZ)|G$FX2O@80j{HeEemu`6%i$F3Q3JbVKKkFSc%?oQ4rIy2>v;k@<~x2G=gQohdVtI}Sjxt%eYe)P|pu+;8k zrsFNC?&4iX8;>ToUMyer{^F}Ijx79Qfp4}4PP%nMM^;QgfX8ZbsPI*b4jv<(g-%HZ zzu#_{XL86$M&HQneEM0|9dGBJ{(D^IjP$-Wg+ljrl64N8vDkM-E@17dYoYETvttc| zDkY~Z-{8!2OV8v8L&nA>9}E_A`kEX{-#k^NsPZ_s-1(h%!{2UbPxFn4`8Qp(c&@o; z^|`sbE`MJ7Jp8=odH3+Cxl2y=`{mu)l6$@R`)1iRwsW@pk8Onh<;@S0iV=$v3lx6C zRq7g(A16~OzSsMI*F(|I_j%-J_^t>j#-8s@>D;dt3`=JfFSb)X8hshe$?`zts4d;XqPihpm4dngHZ&blF&TsXOZo43<#NyRps zJ<>c9=B|v3-(9>Xz2o2p*D9aw;k(T@XnQR-oOJrd(p5@#pEyP{m9H#w-MgyV_kZKN z4gbWyo^Nb4JZSjv+lw1(3aSG5EI)QCQsy}C$A?o)Bw%+GV~lFKozZ2P0zH!q&(pS)pSioJ%(thdd* zm1(~hB-A7vGTeOT$hr8`%I7!qe;4SMmMghy1hdU_ntJ3&LHm5s$z1lE3mFfzT?q7f z$tUdm&Tnm=j{dXg&p*yHp5B+BxpWS{e2Tj^>qf`s8y_6YOvL{0jhD3dGJBXgsfFkG zk>Gs{;rxku(aacx(QTE%E zge1kqCkm|Y81(gU|7`i=wIxf_$!ozr~MUyiULQ-lHnw)8J9imsF=AM&S*Ui9j zK{abjnWbX(?r9ZUU8Gaf{&jbFpNLA&Gr#QiS=0FMxt{Gp$w@92<+`7?wNDOysA6+z z%JkQ#wS6^DJrrI!e~OU()D67fo~#H|RGT;1R7XA8OHF4b&!UGjVmJK|3N)OQ6TS8R z#=@dC&%G;8sk(kWY`Y_Sp7p+$n~sHUpSQc0uR15J?BJ`}=W4qrA6xKfyR6F1vsoAO zE-Xr`RNQ5=dz*{*^vt`r^4>bWEsr|6Xzc~PtJ{)WRaRW7cq`P@c;lghjPuE&GmEu6 zmx`5!^uF%7U$llZGHHq1%<%g*AEq>@v($B%X;^y)dr1V}D)0SNdn6Ny~;_YP!I2=Wosq`NbV6zdZ+f_I?MXp z{TE#%qj&D}tUf35Fk<`0j00x`7-zppE1jX4Gi{lha#^Q}_vBQK3nDD@4Lr2o{Sqh* zSl}fu;8373NrLgtSBE_}PQKq6Cq7}tapmh1_-tl=b!(V6=g{7o1QrQDDLny}KRr{+ zgDl-6*>~KLddJPO@yzGcLsJ5Uqk=9>xHRkgsrPep7Bcg^|GZy%Ukz_;J8qJHX?uu%R@_DJwX?3S5|vVHdAn>EgRt?$zz}Ko zm5a3&NiWVUnIOuxq~`vy%1Qq1xwE(nu3y>3b9LcWLA}i$H3A{6)cL@&G&%I^D^b4VZ|6lx6~5dOI=iEh zVdI%yul^ak9y-@pms*+~M> zh317xD;R}34?L51cxiO0&wcfY3489Uu>E0Qc5lJIl{Q6>#1o#~e4ujd#|oc#qiBYs zd6!MTo&FoS)VgG*vLVYFn@xsnK3Q_F6qn3B`t`?8R?*b(xT=#gR=!Mn6#JQbQ@zZr z{Z~KRo-<@#e5;|@XW7?p>>-M)Dtsqt9ucW8e{uKN?n&3wv#%agQs%uYy~FxN=#IK%)lQC&?(A4~Wa>igUh|`WRUKzfj##0^wZo>M zs9<7g%p5B{MNLEZ`MGENPWPSn3s_we)Nw6~&*#$dqeqXf7Jl?*<=J1^z1@e!`7(Xu zME5EEy!YYjN$baD$Jq6hx6jHmyFd9)#U|@e(d$oZHy0Ne#WB@LoIY;FuDapVMnCVS z^)ix2Pk+l$dB#3Tu+X)zoc*5U_nNf;@Z=QKK4DbU#nCV^$Vux%!V!zZN)sjvR4aF+Y^(5El63Jw1BZ2) ztl^!M<8{7eZ?2@AKX=k-qxFQ7dD2b%Z_IC`JiN|((M0)FU!=Z8*A$n^#Tg-iGj65y zZis%oS?P`8G#zi2pC(N%Jl7+?YTZyZi38Xzo>+nzCflmyk(TCuTKf zXIRXW{#ofY`?%qzA40p#1ca+jZH?90RQ2J^0ev&CT`AWqrnv+ctbOQr_?^k0udZC$ z^U}=a;|*1=n0Q(4U-CoGg8M^g+&O+@&U@G2-fFn8=fS}r4*WH{wF{=)TOoh+1m9nc zFK4e?Gr2W>S}%3Vpt<#?(vP|7*4*n|B@`+&le)t{$p#(zxI=%Fz1ZP*-jXU>TlQ_z zJD$cf|3T%WRjTr`|6lw)IB&jl+H|v&6Na5ner~#S`F?Fh-;(7%a}6!jV)@M5MEd7e z`rqCceT_%j`kauUy@t)QU!NY$&a1!9`=(#J#x;NPyW=|lw$*m<7vHEhSTAmBO+eCM|3i-T5Y{|Mn;xrS=r>PunnYX;$RnkymBpq&2bc20%t4r{Uo)0#U)gG5E4oHkii%GZ1ut}YAAj9C)F`mU8 z?Z2~c|J{Brshn@!OT#~h?H6yaPKkWy@=fy1%TFdfU8`(A<{$t{& z*;Q=pj-1HNN^(1VJFU#4=G%vyXpX(>tQ9oXUKV|xV;^65cLl>+Uh@eWh9767|N1m# z#*6+*?f#lF+vdqwe7mPS?`p>Utuq9db$0GpoWEuEI-Z%*n;eCELRrBR_X3=M5$B8<4Z9{jFi`Iz+J-@$$Bu71gTAywq%Y_h+8$>&^- z=Wm}sy_Qoq{g`J|?PWQ`riq7^T>KsTd3&6rl5X&4y=*VFiL=wr>`Z&jcI%y?d~x5* zJHJ?89&;9|KP$GL`F^}0`-czeiw%n>Z{+fIRIX+Ex8E%8)W3MoIbsJj_O8AlG-K&{ zkto)*sgVn>ewd`?ywG&QinwJLqB6MeEV^Lzk=4#``nz%r-T3$0%Gwep3$4VfGE zLScv13!nG%U&ct;2>kG)(Q``1Ta|_F_3#Ub+TEED& zh8>=Bp{n(oLqtT2Q}#jwrO1V)R|IQ37BVM9xHLPxPONa*?ZnQt*@<7PM1ZHaqW^Dc zS)jw+CZ9$wU8gCX1+)IjYm_ciX%$mEByu)W^xA5l=^9%&7A{Mi6K1@BhMKm?uR|vD zE=#SsK3hvPie>XlFTZTj|GzTk*Vk@2zu)fG^LY8UpV#+)`+42{_n+5`YyW+{XopMs zbcXEqtMf%>-CWgpnK42>D{B1(B_5--&$i6Fy(Dt1K%QiE5F!e0q-A zhdL|KSzjMN=6`Hvzw*MFXZ6pwOnm(~G-KI${yBWds}@@ZEp(l5*6zQXLEUGKR=2uh zspZO+|4mxc{!A~jnsu~cXXHXdwll?+vZue!Tyw!ke*SvhIrUqejQ_p1oO>YYz#dPK zba{c*%$vvm^f)c9J#*V)?y=vM!3!hp&l^TvKL2l4-M8ysW=~&sf6te{KfZ#{pGDt) zr&{T>xfM8>S}YLS$kVUv(=O%b-@>-SJtJ^OibVfXwIw$%H-6PSr}4P*VL2Qp92?(eGHj0v)gJ*!OE34 z;*Fhzb&jNqUSBA8drrXh)tvmH z!tSKiJ>}Vsyjk(bt4k8Sxu36lm~6OuyN0sKXUVXln_~Ck&Q~YSczNYOX@O>O=960E zUag&BXKPHZ9s04_;@ivnr`#w102zlF`o^$?sQ?1p`(3p2$ZrQ{vCpfBZ4?{_E%eGw*%>nYL~c?_96G*QZ1G#ic)(r{K&veY)bresjV3lMCNu z$?dIMz41rRGqLr7RnlCOC*)hbQ*h;YTzqBcutuJuBD1Wbi^8D{oqW{0GZNGo(`1bdv3wUQ$v|K63`(N|t=6(6# z;)>TvPx9Z&Zr)_x|10PEsnp+}<_AYA>Ue*BniIdSK=bF$$dmHLmHOxI{W+Zzy{zY$ z(Mc!O{PQpG{;Wyf_NOuRPvEu{=E7D}ufBZ~Ywv3lzExxA;;7EQ+&53}e>C&xoWE5s zPfP5VUGLSYFLt`PLhM_nt@}rxy_Y_>+Dti;Zh9eT-<$upSNwQ;;-Aeaqh-zWvt%r? z)+W78Q;q+7?}l^bo9o|CzwEAm%)Pv5(aZfsHP)@o*;h{Jp8lpM9-sgJ-t$dD-CzHH zT2UXf?bqrZOJo0*tyxyLeBG6}cgyMuZNvqza3c;7w#*)`~UOK?7y2k z?Zv+rnklT;Kfk4@d-K2Ss^^a{udXk-u6grzdj0)XSs{-W>!0TDS!h2iFuum;LGk~! z&;LBP*|y(u>g5IbX8Sg$s(#P#7d|r&%v`9<+ znb;lezqT7<)#S4;H@uystjJwdkUr(3C69Mz#q$$RCm71k&tp8wmCm+}@9;F8)^m*y z-|(v|^)%${nQxG=_xguoPvySpeeYfW*iL!>o;kcm^?c3#?fwR<8rGk4-COwZHfyZc zwp1(Mr;@j}xBCU1i{d?{Id851eZN~rqGRqbU;c1&d;W$MN!zu{l777Z;`jW|oAs}j zKi=G$&hcz@$-eJro&`R#_UC!~^v{0TL_vGz9~E=|t1td>`_lRDG7v8Qp;^NF5I_}b@h{vyuQ7>zI|fh&U^1fL|caF6>PEi#~W<*+GpOifZM0CWs(bcRNc!f@nF}{()i~F7Z|*C-v?U_1Y=OYF-K(Bz zrEkv8t6eqc{*$6Br>35GT9B9=>pVGlUFO>2E4yb!KRG4kK6&vbR@Pb3F;mYLdG{`k zJGSj;k@MC!hXSQ$&j~%}m0n!z9K89>iuIFZZ`5a}_Pz<)Rdn^V?%J<@l@nJlWji`I z;&#B%C6ckP4;TjAYdrPib(EOp>*yH2C+l1mZO!}`Fmv{j#D$wrHeSxoEV7DvwV{(U zWZNT=**6o;@?Kq`7QS%T)@@>*>$LgeuQxRBc(poecJz;s+0kOxt$s&0SuUU1`zU8s z@zvtKWoo>_Y4 zhV*QyS5g)E&VM&r-7?==$h$Rkarz%T0m-mdLvxT=*@=S}lm!qG& z^nLNy%KdVU*>n4@+Bs)qc%0SUWcsyI<{J9y6>d4v`RPmNrzfvoEsQZu&)r-c$h(Pm zu1V#umPMS8KQH&6VdFV3MkU@bvF72D%+{iROM|(&)y4c~$*thN;=A54{YUOunZs7c zUOwCty+gk0>%pzp4Yr@W7_oUq|KUUPBeTvWRTesHPhOK^*mY~`weXKW4zD;c;nBq2 zz=ww_+{%vMQCmLw==*@3JV&!+By4z#Z#;57IgRm@!DQVe#}_W;)-&s8&6%nzA$v9B zuif4mkENekS#tCD#95YT9Q!lpn%Enut2qZsZ{$09o~>JI@Tc1MqV}1jkhm8sa!bz~ z+)-kZ9P@R$oYwgj&nND?Vb48h@*|lT5x$)}FGjvxRTlcM?3q#2#(BpzCYOeU?wrwY zST?KTl~?Lj-#goE)aNV@S-bk&QMH!lf_<*MzMp5;EIp&6NTgI{N@OMILV zZ&!Rhv10MR7psmP-K76Sl$%$k<94!B>HQ}W=b|sX%6WD;VcVJ#v#Z+189`s3ojGgD zyLM}+#A@!?bFTYTzR!MXZ1-h}@MpKs_@p;;l~*0TT$t)TzyIbDnXYSB#otX^eMfsi z3incFnd29YW*MHDF-?E@$!7Por#usj+j@OYpEz~mLRPv+a_X@SiII^%&PdKve%^CJ zZmO+TK=|cLmn^JoYcDQxb$+`~ z7R#RdKdxo!B~k05cZ~AWeICi%ntarrk?i2jy=9WaooD_$E7m?ce(ayA>#5_ue_yqP z9;$n&`p~lG-Hrzt>+biIu?FqSc4W@amAz1}sP>Hbt>b*Fz6VGQIHsCq-zK%bH z=F6qs*Xvm3EzG(5YKD1Yr2E=2xzoeE$QhZ1&9xZ~u|S7WV&i zzGtb-mGbw&TLm^+o$r2b6&R>Al__(x<0Q+8Dq0gx$zBQ*lI@WCvZrKv`1;!!N1127 zJ(=o#>yy`wrZuPCxy(fFBnEW7W|3GjQT3$i*15^Y(A?j zQ|AufJGJQ{FMqRMbeMGjGt?`JQq#B$6pT%{4B$Y)%+%D_R3S|PB4%o7U}2&FmQ~1u ziCGvKnxcvs8kiVc7@&$7m|7Z|p{X-4wJSel!dqq@bwz|z73&CLde1_p+f=;{niP0`$JU}#`yhHi(Ep@{{0xEL8)n4p_y zVs2oBZk~y`iHRYay{6_CM(AN+XkdmB_J#%)nD!c(8l(Hy$jBT+uZg*(5xSdAEDTMN z+)`4Mn3R9uo+R02-fhDOGQMn+t!s;>TS E0A}S*2><{9 literal 0 HcmV?d00001 From 9ef00d187f3b96312488166d1cb9edbaa0df2c6d Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 29 Dec 2014 22:55:58 -0800 Subject: [PATCH 276/374] %s/master/manager/g + better comments --- uncore/src/main/scala/cache.scala | 20 +- uncore/src/main/scala/coherence.scala | 255 +++++++++++++------------- uncore/src/main/scala/htif.scala | 4 +- uncore/src/main/scala/network.scala | 2 +- uncore/src/main/scala/tilelink.scala | 142 ++++++++------ uncore/src/main/scala/uncore.scala | 12 +- 6 files changed, 226 insertions(+), 209 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index d871e652..0e963f24 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -118,7 +118,7 @@ trait HasL2InternalRequestState extends L2HellaCacheBundle { } object L2Metadata { - def apply(tag: Bits, coh: MasterMetadata) = { + def apply(tag: Bits, coh: ManagerMetadata) = { val meta = new L2Metadata meta.tag := tag meta.coh := coh @@ -126,7 +126,7 @@ object L2Metadata { } } class L2Metadata extends Metadata with L2HellaCacheParameters { - val coh = new MasterMetadata + val coh = new ManagerMetadata } class L2MetaReadReq extends MetaReadReq with HasL2Id { @@ -156,7 +156,7 @@ class L2MetaRWIO extends L2HellaCacheBundle with HasL2MetaReadIO with HasL2MetaW class L2MetadataArray extends L2HellaCacheModule { val io = new L2MetaRWIO().flip - val meta = Module(new MetadataArray(() => L2Metadata(UInt(0), co.masterMetadataOnFlush))) + val meta = Module(new MetadataArray(() => L2Metadata(UInt(0), co.managerMetadataOnFlush))) meta.io.read <> io.read meta.io.write <> io.write @@ -322,7 +322,7 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac // Wire finished transaction acks val finish = io.inner.finish - val finish_idx = finish.bits.payload.master_xact_id + val finish_idx = finish.bits.payload.manager_xact_id trackerList.zipWithIndex.map { case (t, i) => t.io.inner.finish.valid := finish.valid && finish_idx === UInt(i) } @@ -355,7 +355,7 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac class L2WritebackReq extends L2HellaCacheBundle with HasL2Id { val addr = UInt(width = tlAddrBits) - val coh = new MasterMetadata + val coh = new ManagerMetadata val way_en = Bits(width = nWays) } @@ -385,7 +385,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str val state = Reg(init=s_idle) val xact_addr = Reg(io.inner.acquire.bits.payload.addr.clone) - val xact_coh = Reg{ new MasterMetadata } + val xact_coh = Reg{ new ManagerMetadata } val xact_way_en = Reg{ Bits(width = nWays) } val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } val xact_id = Reg{ UInt() } @@ -406,7 +406,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && (state === s_probe) - val next_coh_on_rel = co.masterMetadataOnRelease(c_rel.payload, xact_coh, c_rel.header.src) + val next_coh_on_rel = co.managerMetadataOnRelease(c_rel.payload, xact_coh, c_rel.header.src) io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.payload := Bundle(UncachedWrite(xact_addr, @@ -577,7 +577,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.meta.write.bits.idx := xact_addr(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en io.meta.write.bits.data.tag := xact_addr >> UInt(idxBits) - io.meta.write.bits.data.coh := co.masterMetadataOnRelease(xact, + io.meta.write.bits.data.coh := co.managerMetadataOnRelease(xact, xact_meta.coh, xact_src) io.wb.req.valid := Bool(false) @@ -701,8 +701,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St (xact.addr === c_rel.payload.addr) && (state === s_probe) - val next_coh_on_rel = co.masterMetadataOnRelease(c_rel.payload, xact_meta.coh, c_rel.header.src) - val next_coh_on_gnt = co.masterMetadataOnGrant(c_gnt.payload, xact_meta.coh, + val next_coh_on_rel = co.managerMetadataOnRelease(c_rel.payload, xact_meta.coh, c_rel.header.src) + val next_coh_on_gnt = co.managerMetadataOnGrant(c_gnt.payload, xact_meta.coh, c_gnt.header.dst) val outer_write = Bundle(UncachedWrite(xact_addr, UInt(trackerId), xact_data(outer_data_write_cnt)), diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index c7999f4b..08d7cf20 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -3,6 +3,8 @@ package uncore import Chisel._ + +// Classes to represent coherence information in clients and managers abstract class CoherenceMetadata extends Bundle with CoherenceAgentParameters class ClientMetadata extends CoherenceMetadata { @@ -18,21 +20,23 @@ object ClientMetadata { } } -class MasterMetadata extends CoherenceMetadata { +class ManagerMetadata extends CoherenceMetadata { val state = UInt(width = co.masterStateWidth) val sharers = UInt(width = co.dir.width) - override def clone = new MasterMetadata().asInstanceOf[this.type] + override def clone = new ManagerMetadata().asInstanceOf[this.type] } -object MasterMetadata { +object ManagerMetadata { def apply(state: UInt, sharers: UInt) = { - val m = new MasterMetadata + val m = new ManagerMetadata m.state := state m.sharers := sharers m } - def apply(state: UInt): MasterMetadata = apply(state, new MasterMetadata().co.dir.flush) + def apply(state: UInt): ManagerMetadata = apply(state, new ManagerMetadata().co.dir.flush) } +// This class encapsulates transformations on different directory information +// storage formats abstract class DirectoryRepresentation(val width: Int) { def pop(prev: UInt, id: UInt): UInt def push(prev: UInt, id: UInt): UInt @@ -66,15 +70,16 @@ class FullRepresentation(nClients: Int) extends DirectoryRepresentation(nClients def full(s: UInt) = s } +// Coherence policy inferface for clients and managers abstract class CoherencePolicy(val dir: DirectoryRepresentation) { def nClientStates: Int - def nMasterStates: Int + def nManagerStates: Int def nAcquireTypes: Int def nProbeTypes: Int def nReleaseTypes: Int def nGrantTypes: Int def clientStateWidth = log2Up(nClientStates) - def masterStateWidth = log2Up(nMasterStates) + def masterStateWidth = log2Up(nManagerStates) def acquireTypeWidth = log2Up(nAcquireTypes) def probeTypeWidth = log2Up(nProbeTypes) def releaseTypeWidth = log2Up(nReleaseTypes) @@ -82,49 +87,48 @@ abstract class CoherencePolicy(val dir: DirectoryRepresentation) { def isHit (cmd: UInt, m: ClientMetadata): Bool def isValid (m: ClientMetadata): Bool - def isHit (incoming: Acquire, m: MasterMetadata): Bool - def isValid (m: MasterMetadata): Bool + def isHit (incoming: Acquire, m: ManagerMetadata): Bool + def isValid (m: ManagerMetadata): Bool def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool def needsWriteback(m: ClientMetadata): Bool - def needsWriteback(m: MasterMetadata): Bool + def needsWriteback(m: ManagerMetadata): Bool def clientMetadataOnHit(cmd: UInt, m: ClientMetadata): ClientMetadata def clientMetadataOnCacheControl(cmd: UInt): ClientMetadata def clientMetadataOnFlush: ClientMetadata def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire): ClientMetadata def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata): ClientMetadata - def masterMetadataOnFlush: MasterMetadata - def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt): MasterMetadata - def masterMetadataOnGrant(outgoing: Grant, m: MasterMetadata, dst: UInt): MasterMetadata + def managerMetadataOnFlush: ManagerMetadata + def managerMetadataOnRelease(incoming: Release, m: ManagerMetadata, src: UInt): ManagerMetadata + def managerMetadataOnGrant(outgoing: Grant, m: ManagerMetadata, dst: UInt): ManagerMetadata def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt - def getProbeType(a: Acquire, m: MasterMetadata): UInt + def getProbeType(a: Acquire, m: ManagerMetadata): UInt def getProbeTypeOnVoluntaryWriteback: UInt def getReleaseTypeOnCacheControl(cmd: UInt): UInt def getReleaseTypeOnVoluntaryWriteback(): UInt def getReleaseTypeOnProbe(p: Probe, m: ClientMetadata): UInt - def getGrantType(a: Acquire, m: MasterMetadata): UInt - def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt + def getGrantType(a: Acquire, m: ManagerMetadata): UInt + def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt - def messageHasData(rel: SourcedMessage): Bool + def messageHasData(rel: TileLinkChannel): Bool def messageUpdatesDataArray(g: Grant): Bool def isVoluntary(rel: Release): Bool def isVoluntary(gnt: Grant): Bool - def requiresOuterRead(acq: Acquire, m: MasterMetadata): Bool - def requiresOuterWrite(acq: Acquire, m: MasterMetadata): Bool + def requiresOuterRead(acq: Acquire, m: ManagerMetadata): Bool + def requiresOuterWrite(acq: Acquire, m: ManagerMetadata): Bool def requiresSelfProbe(a: Acquire): Bool - def requiresProbes(a: Acquire, m: MasterMetadata): Bool - def requiresProbesOnVoluntaryWriteback(m: MasterMetadata): Bool + def requiresProbes(a: Acquire, m: ManagerMetadata): Bool + def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata): Bool def requiresAckForGrant(g: Grant): Bool def requiresAckForRelease(r: Release): Bool - def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool - def getGrantTypeForUncached(a: Acquire, m: MasterMetadata): UInt = { + def getGrantTypeForUncached(a: Acquire, m: ManagerMetadata): UInt = { MuxLookup(a.a_type, Grant.uncachedRead, Array( Acquire.uncachedRead -> Grant.uncachedRead, Acquire.uncachedWrite -> Grant.uncachedWrite, @@ -135,14 +139,14 @@ abstract class CoherencePolicy(val dir: DirectoryRepresentation) { class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 2 - def nMasterStates = 2 + def nManagerStates = 2 def nAcquireTypes = 1 def nProbeTypes = 2 def nReleaseTypes = 5 def nGrantTypes = 2 val clientInvalid :: clientValid :: Nil = Enum(UInt(), nClientStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) + val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) val acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) @@ -154,8 +158,8 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isHit (cmd: UInt, m: ClientMetadata): Bool = isValid(m) def isValid (m: ClientMetadata): Bool = m.state != clientInvalid - def isHit (incoming: Acquire, m: MasterMetadata): Bool = isValid(m) - def isValid (m: MasterMetadata): Bool = m.state != masterInvalid + def isHit (incoming: Acquire, m: ManagerMetadata): Bool = isValid(m) + def isValid (m: ManagerMetadata): Bool = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = (outstanding.a_type != acquireReadExclusive) def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { @@ -167,7 +171,7 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def needsWriteback (m: ClientMetadata): Bool = { needsTransactionOnCacheControl(M_INV, m) } - def needsWriteback(m: MasterMetadata) = isValid(m) + def needsWriteback(m: ManagerMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = m def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( @@ -183,18 +187,18 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeInvalidate -> clientInvalid, probeCopy -> m.state ))) - def masterMetadataOnFlush = MasterMetadata(masterInvalid) - def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, dir.pop(m.sharers, src)) + def managerMetadataOnFlush = ManagerMetadata(masterInvalid) + def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(masterValid, dir.pop(m.sharers, src)) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, r.is(releaseInvalidateAck) -> next )) } - def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = MasterMetadata(masterValid, m.sharers) + def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { + val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) + val uncached = ManagerMetadata(masterValid, m.sharers) Mux(g.uncached, uncached, cached) } @@ -217,7 +221,7 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { Mux(needsWriteback(m), with_data, without_data) } - def messageHasData(msg: SourcedMessage) = msg match { + def messageHasData(msg: TileLinkChannel) = msg match { case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) @@ -230,12 +234,12 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: MasterMetadata): UInt = + def getGrantType(a: Acquire, m: ManagerMetadata): UInt = Mux(a.uncached, getGrantTypeForUncached(a, m), grantReadExclusive) - def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck + def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck - def getProbeType(a: Acquire, m: MasterMetadata): UInt = { + def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( Acquire.uncachedRead -> probeCopy, @@ -245,29 +249,28 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { } def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate - def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) - def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) - def requiresProbesOnVoluntaryWriteback(m: MasterMetadata) = !dir.none(m.sharers) - def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) + def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) + def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) } class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 3 - def nMasterStates = 2 + def nManagerStates = 2 def nAcquireTypes = 2 def nProbeTypes = 3 def nReleaseTypes = 7 def nGrantTypes = 2 val clientInvalid :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) + val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) @@ -279,8 +282,8 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isHit (cmd: UInt, m: ClientMetadata) = isValid(m) def isValid (m: ClientMetadata) = m.state != clientInvalid - def isHit (incoming: Acquire, m: MasterMetadata) = isValid(m) - def isValid (m: MasterMetadata) = m.state != masterInvalid + def isHit (incoming: Acquire, m: ManagerMetadata) = isValid(m) + def isValid (m: ManagerMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && outstanding.uncached) || @@ -295,7 +298,7 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def needsWriteback (m: ClientMetadata): Bool = { needsTransactionOnCacheControl(M_INV, m) } - def needsWriteback(m: MasterMetadata) = isValid(m) + def needsWriteback(m: ManagerMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state)) @@ -316,18 +319,18 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeDowngrade -> clientExclusiveClean, probeCopy -> m.state ))) - def masterMetadataOnFlush = MasterMetadata(masterInvalid) - def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, dir.pop(m.sharers,src)) + def managerMetadataOnFlush = ManagerMetadata(masterInvalid) + def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(masterValid, dir.pop(m.sharers,src)) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, r.is(releaseInvalidateAck) -> next )) } - def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = MasterMetadata(masterValid, m.sharers) + def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { + val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) + val uncached = ManagerMetadata(masterValid, m.sharers) Mux(g.uncached, uncached, cached) } @@ -356,7 +359,7 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { Mux(needsWriteback(m), with_data, without_data) } - def messageHasData(msg: SourcedMessage) = msg match { + def messageHasData(msg: TileLinkChannel) = msg match { case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) @@ -369,12 +372,12 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: MasterMetadata): UInt = { + def getGrantType(a: Acquire, m: ManagerMetadata): UInt = { Mux(a.uncached, getGrantTypeForUncached(a, m), grantReadExclusive) } - def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck + def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck - def getProbeType(a: Acquire, m: MasterMetadata): UInt = { + def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( Acquire.uncachedRead -> probeCopy, @@ -384,31 +387,29 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { } def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate - def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) - def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) - def requiresProbesOnVoluntaryWriteback(m: MasterMetadata) = !dir.none(m.sharers) - - def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) + def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) + def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) } class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 3 - def nMasterStates = 2 + def nManagerStates = 2 def nAcquireTypes = 2 def nProbeTypes = 3 def nReleaseTypes = 7 def nGrantTypes = 3 val clientInvalid :: clientShared :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) + //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) + val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) val acquireReadShared :: acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) @@ -425,8 +426,8 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isValid (m: ClientMetadata): Bool = { m.state != clientInvalid } - def isHit (incoming: Acquire, m: MasterMetadata) = isValid(m) - def isValid (m: MasterMetadata) = m.state != masterInvalid + def isHit (incoming: Acquire, m: ManagerMetadata) = isValid(m) + def isValid (m: ManagerMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && outstanding.uncached) || @@ -441,7 +442,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def needsWriteback (m: ClientMetadata): Bool = { needsTransactionOnCacheControl(M_INV, m) } - def needsWriteback(m: MasterMetadata) = isValid(m) + def needsWriteback(m: ManagerMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state)) @@ -461,18 +462,18 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeDowngrade -> clientShared, probeCopy -> m.state ))) - def masterMetadataOnFlush = MasterMetadata(masterInvalid) - def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, dir.pop(m.sharers,src)) + def managerMetadataOnFlush = ManagerMetadata(masterInvalid) + def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(masterValid, dir.pop(m.sharers,src)) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, r.is(releaseInvalidateAck) -> next )) } - def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = MasterMetadata(masterValid, m.sharers) + def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { + val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) + val uncached = ManagerMetadata(masterValid, m.sharers) Mux(g.uncached, uncached, cached) } @@ -501,7 +502,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { Mux(needsWriteback(m), with_data, without_data) } - def messageHasData(msg: SourcedMessage) = msg match { + def messageHasData(msg: TileLinkChannel) = msg match { case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) @@ -514,15 +515,15 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: MasterMetadata): UInt = { + def getGrantType(a: Acquire, m: ManagerMetadata): UInt = { Mux(a.uncached, getGrantTypeForUncached(a, m), Mux(a.a_type === acquireReadShared, Mux(!dir.none(m.sharers), grantReadShared, grantReadExclusive), grantReadExclusive)) } - def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck + def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck - def getProbeType(a: Acquire, m: MasterMetadata): UInt = { + def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( Acquire.uncachedRead -> probeCopy, @@ -536,34 +537,32 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { } def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate - def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) - def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) && + def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) && Mux(dir.one(m.sharers), Bool(true), Mux(a.uncached, a.a_type != Acquire.uncachedRead, a.a_type != acquireReadShared)) - def requiresProbesOnVoluntaryWriteback(m: MasterMetadata) = !dir.none(m.sharers) - - def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) + def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) } class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 4 - def nMasterStates = 2 + def nManagerStates = 2 def nAcquireTypes = 2 def nProbeTypes = 3 def nReleaseTypes = 7 def nGrantTypes = 4 val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) + //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) + val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) val acquireReadShared :: acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) @@ -580,8 +579,8 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isValid (m: ClientMetadata): Bool = { m.state != clientInvalid } - def isHit (incoming: Acquire, m: MasterMetadata) = isValid(m) - def isValid (m: MasterMetadata) = m.state != masterInvalid + def isHit (incoming: Acquire, m: ManagerMetadata) = isValid(m) + def isValid (m: ManagerMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && outstanding.uncached) || @@ -596,7 +595,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def needsWriteback (m: ClientMetadata): Bool = { needsTransactionOnCacheControl(M_INV, m) } - def needsWriteback(m: MasterMetadata) = isValid(m) + def needsWriteback(m: ManagerMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state)) @@ -621,18 +620,18 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeDowngrade -> clientShared, probeCopy -> m.state ))) - def masterMetadataOnFlush = MasterMetadata(masterInvalid) - def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, dir.pop(m.sharers,src)) + def managerMetadataOnFlush = ManagerMetadata(masterInvalid) + def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(masterValid, dir.pop(m.sharers,src)) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, r.is(releaseInvalidateAck) -> next )) } - def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = MasterMetadata(masterValid, m.sharers) + def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { + val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) + val uncached = ManagerMetadata(masterValid, m.sharers) Mux(g.uncached, uncached, cached) } @@ -661,7 +660,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { Mux(needsWriteback(m), with_data, without_data) } - def messageHasData(msg: SourcedMessage) = msg match { + def messageHasData(msg: TileLinkChannel) = msg match { case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) @@ -674,15 +673,15 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: MasterMetadata): UInt = { + def getGrantType(a: Acquire, m: ManagerMetadata): UInt = { Mux(a.uncached, getGrantTypeForUncached(a, m), Mux(a.a_type === acquireReadShared, Mux(!dir.none(m.sharers), grantReadShared, grantReadExclusive), grantReadExclusive)) } - def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck + def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck - def getProbeType(a: Acquire, m: MasterMetadata): UInt = { + def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( Acquire.uncachedRead -> probeCopy, @@ -696,34 +695,32 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { } def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate - def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) - def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) && + def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) && Mux(dir.one(m.sharers), Bool(true), Mux(a.uncached, a.a_type != Acquire.uncachedRead, a.a_type != acquireReadShared)) - def requiresProbesOnVoluntaryWriteback(m: MasterMetadata) = !dir.none(m.sharers) - - def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) + def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) } class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 7 - def nMasterStates = 2 + def nManagerStates = 2 def nAcquireTypes = 3 def nProbeTypes = 4 def nReleaseTypes = 11 def nGrantTypes = 5 val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: clientSharedByTwo :: clientMigratoryClean :: clientMigratoryDirty :: Nil = Enum(UInt(), nClientStates) - //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) + //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) + val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) val acquireReadShared :: acquireReadExclusive :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(UInt(), nProbeTypes) @@ -739,8 +736,8 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d def isValid (m: ClientMetadata): Bool = { m.state != clientInvalid } - def isHit (incoming: Acquire, m: MasterMetadata) = isValid(m) - def isValid (m: MasterMetadata) = m.state != masterInvalid + def isHit (incoming: Acquire, m: ManagerMetadata) = isValid(m) + def isValid (m: ManagerMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && outstanding.uncached) || @@ -755,7 +752,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d def needsWriteback (m: ClientMetadata): Bool = { needsTransactionOnCacheControl(M_INV, m) } - def needsWriteback(m: MasterMetadata) = isValid(m) + def needsWriteback(m: ManagerMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata( Mux(isWrite(cmd), MuxLookup(m.state, clientExclusiveDirty, Array( @@ -792,9 +789,9 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d clientMigratoryClean -> clientSharedByTwo, clientMigratoryDirty -> clientInvalid)) ))) - def masterMetadataOnFlush = MasterMetadata(masterInvalid) - def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, dir.pop(m.sharers,src)) + def managerMetadataOnFlush = ManagerMetadata(masterInvalid) + def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(masterValid, dir.pop(m.sharers,src)) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -803,9 +800,9 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d r.is(releaseInvalidateAckMigratory) -> next )) } - def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = MasterMetadata(masterValid, m.sharers) + def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { + val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) + val uncached = ManagerMetadata(masterValid, m.sharers) Mux(g.uncached, uncached, cached) } @@ -837,7 +834,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d Mux(needsWriteback(m), with_data, without_data) } - def messageHasData(msg: SourcedMessage) = msg match { + def messageHasData(msg: TileLinkChannel) = msg match { case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) @@ -850,7 +847,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: MasterMetadata): UInt = { + def getGrantType(a: Acquire, m: ManagerMetadata): UInt = { Mux(a.uncached, getGrantTypeForUncached(a, m), MuxLookup(a.a_type, grantReadShared, Array( acquireReadShared -> Mux(!dir.none(m.sharers), grantReadShared, grantReadExclusive), @@ -858,9 +855,9 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI for broadcast? ))) } - def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck + def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck - def getProbeType(a: Acquire, m: MasterMetadata): UInt = { + def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( Acquire.uncachedRead -> probeCopy, @@ -875,19 +872,17 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d } def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate - def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), acq.a_type != acquireInvalidateOthers) - def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) && + def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) && Mux(dir.one(m.sharers), Bool(true), Mux(a.uncached, a.a_type != Acquire.uncachedRead, a.a_type != acquireReadShared)) - def requiresProbesOnVoluntaryWriteback(m: MasterMetadata) = !dir.none(m.sharers) - - def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) + def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) } diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 378ca3b1..4db133d6 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -134,7 +134,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { val mem_needs_ack = Reg(Bool()) when (io.mem.grant.valid) { mem_acked := Bool(true) - mem_gxid := io.mem.grant.bits.payload.master_xact_id + mem_gxid := io.mem.grant.bits.payload.manager_xact_id mem_gsrc := io.mem.grant.bits.header.src mem_needs_ack := co.requiresAckForGrant(io.mem.grant.bits.payload) } @@ -201,7 +201,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { io.mem.acquire.bits.header.src := UInt(params(LNClients)) // By convention HTIF is the client with the largest id io.mem.acquire.bits.header.dst := UInt(0) // DNC; Overwritten outside module io.mem.finish.valid := (state === state_mem_finish) && mem_needs_ack - io.mem.finish.bits.payload.master_xact_id := mem_gxid + io.mem.finish.bits.payload.manager_xact_id := mem_gxid io.mem.finish.bits.header.dst := mem_gsrc io.mem.probe.ready := Bool(false) io.mem.release.valid := Bool(false) diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index a34e91c1..161a9a0c 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -3,7 +3,7 @@ package uncore import Chisel._ -case object LNMasters extends Field[Int] +case object LNManagers extends Field[Int] case object LNClients extends Field[Int] case object LNEndpoints extends Field[Int] diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 4e5ac961..af10d455 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -4,10 +4,13 @@ package uncore import Chisel._ import scala.math.max -case object TLId extends Field[String] +// Parameters exposed to the top-level design, set based on +// external requirements or design space exploration +// +case object TLId extends Field[String] // Unique name per network case object TLCoherence extends Field[CoherencePolicy] case object TLAddrBits extends Field[Int] -case object TLMasterXactIdBits extends Field[Int] +case object TLManagerXactIdBits extends Field[Int] case object TLClientXactIdBits extends Field[Int] case object TLDataBits extends Field[Int] case object TLDataBeats extends Field[Int] @@ -15,7 +18,7 @@ case object TLDataBeats extends Field[Int] abstract trait TileLinkParameters extends UsesParameters { val tlAddrBits = params(TLAddrBits) val tlClientXactIdBits = params(TLClientXactIdBits) - val tlMasterXactIdBits = params(TLMasterXactIdBits) + val tlManagerXactIdBits = params(TLManagerXactIdBits) val tlDataBits = params(TLDataBits) val tlDataBeats = params(TLDataBeats) val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits @@ -25,13 +28,23 @@ abstract trait TileLinkParameters extends UsesParameters { val tlSubblockUnionBits = max(tlWriteMaskBits, (tlSubblockAddrBits + tlUncachedOperandSizeBits + - tlAtomicOpcodeBits)) + tlAtomicOpcodeBits)) + 1 val co = params(TLCoherence) } abstract class TLBundle extends Bundle with TileLinkParameters abstract class TLModule extends Module with TileLinkParameters +// Directionality of message channel +// Used to hook up logical network ports to physical network ports +trait TileLinkChannel extends TLBundle +trait ClientToManagerChannel extends TileLinkChannel +trait ManagerToClientChannel extends TileLinkChannel +trait ClientToClientChannel extends TileLinkChannel // Unused for now + +// Common signals that are used in multiple channels. +// These traits are useful for type parameterization. +// trait HasPhysicalAddress extends TLBundle { val addr = UInt(width = tlAddrBits) } @@ -40,19 +53,17 @@ trait HasClientTransactionId extends TLBundle { val client_xact_id = Bits(width = tlClientXactIdBits) } -trait HasMasterTransactionId extends TLBundle { - val master_xact_id = Bits(width = tlMasterXactIdBits) +trait HasManagerTransactionId extends TLBundle { + val manager_xact_id = Bits(width = tlManagerXactIdBits) } trait HasTileLinkData extends TLBundle { val data = UInt(width = tlDataBits) } -trait SourcedMessage extends TLBundle -trait ClientSourcedMessage extends SourcedMessage -trait MasterSourcedMessage extends SourcedMessage +// Actual TileLink channel bundle definitions -class Acquire extends ClientSourcedMessage +class Acquire extends ClientToManagerChannel with HasPhysicalAddress with HasClientTransactionId with HasTileLinkData { @@ -61,16 +72,17 @@ class Acquire extends ClientSourcedMessage val subblock = Bits(width = tlSubblockUnionBits) val sbAddrOff = tlSubblockAddrBits + tlUncachedOperandSizeBits val opSzOff = tlUncachedOperandSizeBits + sbAddrOff - def operand_sz(dummy: Int = 0) = subblock(tlUncachedOperandSizeBits-1, 0) - def subblock_addr(dummy: Int = 0) = subblock(sbAddrOff-1, tlUncachedOperandSizeBits) - def atomic_op(dummy: Int = 0) = subblock(opSzOff-1, sbAddrOff) - def write_mask(dummy: Int = 0) = subblock(tlWriteMaskBits-1, 0) + def allocate(dummy: Int = 0) = subblock(0) + def operand_sz(dummy: Int = 0) = subblock(tlUncachedOperandSizeBits, 1) + def subblock_addr(dummy: Int = 0) = subblock(sbAddrOff, tlUncachedOperandSizeBits+1) + def atomic_op(dummy: Int = 0) = subblock(opSzOff, sbAddrOff+1) + def write_mask(dummy: Int = 0) = subblock(tlWriteMaskBits, 1) def is(t: UInt) = a_type === t } object Acquire { val nUncachedAcquireTypes = 3 - //val uncachedRead :: uncachedWrite :: uncachedAtomic :: Nil = Enum(UInt(), nUncachedAcquireTypes) + //TODO: val uncachedRead :: uncachedWrite :: uncachedAtomic :: Nil = Enum(UInt(), nUncachedAcquireTypes) def uncachedRead = UInt(0) def uncachedWrite = UInt(1) def uncachedAtomic = UInt(2) @@ -102,29 +114,29 @@ object Acquire { } object UncachedRead { - def apply(addr: UInt, client_xact_id: UInt, subblock_addr: UInt, operand_sz: UInt): Acquire = { + def apply(addr: UInt, client_xact_id: UInt, subblock_addr: UInt, operand_sz: UInt, alloc: Bool): Acquire = { val acq = Acquire(Acquire.uncachedRead, addr, client_xact_id) acq.uncached := Bool(true) - acq.subblock := Cat(subblock_addr, operand_sz) + acq.subblock := Cat(subblock_addr, operand_sz, alloc) acq } def apply(addr: UInt, client_xact_id: UInt): Acquire = { - apply(addr, client_xact_id, UInt(0), MT_CB) + apply(addr, client_xact_id, UInt(0), MT_CB, Bool(true)) } def apply(addr: UInt): Acquire = { - apply(addr, UInt(0), UInt(0), MT_CB) + apply(addr, UInt(0)) } } object UncachedWrite { - def apply(addr: UInt, client_xact_id: UInt, write_mask: Bits, data: UInt): Acquire = { + def apply(addr: UInt, client_xact_id: UInt, write_mask: Bits, alloc: Bool, data: UInt): Acquire = { val acq = Acquire(Acquire.uncachedWrite, addr, client_xact_id, data) acq.uncached := Bool(true) - acq.subblock := write_mask + acq.subblock := Cat(write_mask, alloc) acq } def apply(addr: UInt, client_xact_id: UInt, data: UInt): Acquire = { - apply(addr, client_xact_id, SInt(-1), data) + apply(addr, client_xact_id, SInt(-1), Bool(true), data) } def apply(addr: UInt, data: UInt): Acquire = { apply(addr, UInt(0), data) @@ -136,11 +148,17 @@ object UncachedAtomic { subblock_addr: UInt, operand_sz: UInt, data: UInt): Acquire = { val acq = Acquire(Acquire.uncachedAtomic, addr, client_xact_id, data) acq.uncached := Bool(true) - acq.subblock := Cat(atomic_opcode, subblock_addr, operand_sz) + acq.subblock := Cat(atomic_opcode, subblock_addr, operand_sz, Bool(true)) acq } } +class Probe extends ManagerToClientChannel + with HasPhysicalAddress { + val p_type = UInt(width = co.probeTypeWidth) + def is(t: UInt) = p_type === t +} + object Probe { def apply(p_type: UInt, addr: UInt) = { val prb = new Probe @@ -150,10 +168,12 @@ object Probe { } } -class Probe extends MasterSourcedMessage - with HasPhysicalAddress { - val p_type = UInt(width = co.probeTypeWidth) - def is(t: UInt) = p_type === t +class Release extends ClientToManagerChannel + with HasPhysicalAddress + with HasClientTransactionId + with HasTileLinkData { + val r_type = UInt(width = co.releaseTypeWidth) + def is(t: UInt) = r_type === t } object Release { @@ -173,18 +193,10 @@ object Release { } } -class Release extends ClientSourcedMessage - with HasPhysicalAddress - with HasClientTransactionId - with HasTileLinkData { - val r_type = UInt(width = co.releaseTypeWidth) - def is(t: UInt) = r_type === t -} - -class Grant extends MasterSourcedMessage +class Grant extends ManagerToClientChannel with HasTileLinkData with HasClientTransactionId - with HasMasterTransactionId { + with HasManagerTransactionId { val uncached = Bool() val g_type = UInt(width = max(log2Up(Grant.nUncachedGrantTypes), co.grantTypeWidth)) def is(t: UInt) = g_type === t @@ -192,29 +204,30 @@ class Grant extends MasterSourcedMessage object Grant { val nUncachedGrantTypes = 3 - //val uncachedRead :: uncachedWrite :: uncachedAtomic :: Nil = Enum(UInt(), nUncachedGrantTypes) + //TODO val uncachedRead :: uncachedWrite :: uncachedAtomic :: Nil = Enum(UInt(), nUncachedGrantTypes) def uncachedRead = UInt(0) def uncachedWrite = UInt(1) def uncachedAtomic = UInt(2) def hasData(g_type: UInt) = Vec(uncachedRead, uncachedAtomic).contains(g_type) - def apply(uncached: Bool, g_type: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt): Grant = { + def apply(uncached: Bool, g_type: UInt, client_xact_id: UInt, manager_xact_id: UInt, data: UInt): Grant = { val gnt = new Grant gnt.uncached := uncached gnt.g_type := g_type gnt.client_xact_id := client_xact_id - gnt.master_xact_id := master_xact_id + gnt.manager_xact_id := manager_xact_id gnt.data := data gnt } - def apply(uncached: Bool, g_type: UInt, client_xact_id: UInt, master_xact_id: UInt): Grant = { - apply(uncached, g_type, client_xact_id, master_xact_id, UInt(0)) + def apply(uncached: Bool, g_type: UInt, client_xact_id: UInt, manager_xact_id: UInt): Grant = { + apply(uncached, g_type, client_xact_id, manager_xact_id, UInt(0)) } } -class Finish extends ClientSourcedMessage with HasMasterTransactionId +class Finish extends ClientToManagerChannel with HasManagerTransactionId +// Complete IO definitions for two types of TileLink clients class UncachedTileLinkIO extends Bundle { val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip @@ -226,6 +239,8 @@ class TileLinkIO extends UncachedTileLinkIO { val release = new DecoupledIO(new LogicalNetworkIO(new Release)) } +// Converts UncachedTileLinkIO to regular TileLinkIO by pinning +// probe.ready and release.valid low class TileLinkIOWrapper extends TLModule { val io = new Bundle { val in = new UncachedTileLinkIO().flip @@ -245,13 +260,16 @@ object TileLinkIOWrapper { } } +// Utility functions for constructing TileLinkIO arbiters abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule { - type MasterSourcedWithId = MasterSourcedMessage with HasClientTransactionId - type ClientSourcedWithId = ClientSourcedMessage with HasClientTransactionId + type ManagerSourcedWithId = ManagerToClientChannel with HasClientTransactionId + type ClientSourcedWithId = ClientToManagerChannel with HasClientTransactionId + // These are filled in depending on whether the arbiter mucks with the + // client ids and then needs to revert them on the way back def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int): Bits - def masterSourcedClientXactId(in: MasterSourcedWithId): Bits - def arbIdx(in: MasterSourcedWithId): UInt + def managerSourcedClientXactId(in: ManagerSourcedWithId): Bits + def arbIdx(in: ManagerSourcedWithId): UInt def hookupClientSource[M <: ClientSourcedWithId] (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], @@ -267,7 +285,7 @@ abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule { }} } - def hookupMasterSource[M <: MasterSourcedWithId] + def hookupManagerSource[M <: ManagerSourcedWithId] (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], out: DecoupledIO[LogicalNetworkIO[M]]) { out.ready := Bool(false) @@ -278,7 +296,7 @@ abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule { out.ready := ins(i).ready } ins(i).bits := out.bits - ins(i).bits.payload.client_xact_id := masterSourcedClientXactId(out.bits.payload) + ins(i).bits.payload.client_xact_id := managerSourcedClientXactId(out.bits.payload) } } } @@ -291,7 +309,7 @@ abstract class UncachedTileLinkIOArbiter(n: Int) } hookupClientSource(io.in.map(_.acquire), io.out.acquire) - hookupMasterSource(io.in.map(_.grant), io.out.grant) + hookupManagerSource(io.in.map(_.grant), io.out.grant) val finish_arb = Module(new RRArbiter(new LogicalNetworkIO(new Finish), n)) io.out.finish <> finish_arb.io.out @@ -306,7 +324,7 @@ abstract class TileLinkIOArbiter(n: Int) extends TileLinkArbiterLike(n) { hookupClientSource(io.in.map(_.acquire), io.out.acquire) hookupClientSource(io.in.map(_.release), io.out.release) - hookupMasterSource(io.in.map(_.grant), io.out.grant) + hookupManagerSource(io.in.map(_.grant), io.out.grant) io.in.map{ _.probe.valid := io.out.probe.valid } io.in.map{ _.probe.bits := io.out.probe.bits } @@ -317,35 +335,39 @@ abstract class TileLinkIOArbiter(n: Int) extends TileLinkArbiterLike(n) { finish_arb.io.in zip io.in map { case (arb, req) => arb <> req.finish } } +// Appends the port index of the arbiter to the client_xact_id abstract trait AppendsArbiterId { val arbN: Int - def clientSourcedClientXactId(in: ClientSourcedMessage with HasClientTransactionId, id: Int) = + def clientSourcedClientXactId(in: ClientToManagerChannel with HasClientTransactionId, id: Int) = Cat(in.client_xact_id, UInt(id, log2Up(arbN))) - def masterSourcedClientXactId(in: MasterSourcedMessage with HasClientTransactionId) = + def managerSourcedClientXactId(in: ManagerToClientChannel with HasClientTransactionId) = in.client_xact_id >> UInt(log2Up(arbN)) - def arbIdx(in: MasterSourcedMessage with HasClientTransactionId) = + def arbIdx(in: ManagerToClientChannel with HasClientTransactionId) = in.client_xact_id(log2Up(arbN)-1,0).toUInt } +// Uses the client_xact_id as is (assumes it has been set to port index) abstract trait PassesId { - def clientSourcedClientXactId(in: ClientSourcedMessage with HasClientTransactionId, id: Int) = + def clientSourcedClientXactId(in: ClientToManagerChannel with HasClientTransactionId, id: Int) = in.client_xact_id - def masterSourcedClientXactId(in: MasterSourcedMessage with HasClientTransactionId) = + def managerSourcedClientXactId(in: ManagerToClientChannel with HasClientTransactionId) = in.client_xact_id - def arbIdx(in: MasterSourcedMessage with HasClientTransactionId) = + def arbIdx(in: ManagerToClientChannel with HasClientTransactionId) = in.client_xact_id } +// Overwrites some default client_xact_id with the port idx abstract trait UsesNewId { val arbN: Int - def clientSourcedClientXactId(in: ClientSourcedMessage with HasClientTransactionId, id: Int) = + def clientSourcedClientXactId(in: ClientToManagerChannel with HasClientTransactionId, id: Int) = UInt(id, log2Up(arbN)) - def masterSourcedClientXactId(in: MasterSourcedMessage with HasClientTransactionId) = + def managerSourcedClientXactId(in: ManagerToClientChannel with HasClientTransactionId) = UInt(0) - def arbIdx(in: MasterSourcedMessage with HasClientTransactionId) = + def arbIdx(in: ManagerToClientChannel with HasClientTransactionId) = in.client_xact_id } +// Mix-in id generation traits to make concrete arbiter classes class UncachedTileLinkIOArbiterThatAppendsArbiterId(val n: Int) extends UncachedTileLinkIOArbiter(n) with AppendsArbiterId class UncachedTileLinkIOArbiterThatPassesId(val n: Int) extends UncachedTileLinkIOArbiter(n) with PassesId class UncachedTileLinkIOArbiterThatUsesNewId(val n: Int) extends UncachedTileLinkIOArbiter(n) with UsesNewId diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index e8e82d22..d2531a9e 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -194,7 +194,7 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := xact_src io.inner.grant.bits.payload := Grant(Bool(false), - co.getGrantTypeOnVoluntaryWriteback(co.masterMetadataOnFlush), + co.getGrantTypeOnVoluntaryWriteback(co.managerMetadataOnFlush), xact_client_xact_id, UInt(trackerId)) @@ -261,7 +261,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val curr_p_id = PriorityEncoder(probe_flags) val pending_outer_write = co.messageHasData(xact) - val pending_outer_read = co.requiresOuterRead(xact, co.masterMetadataOnFlush) + val pending_outer_read = co.requiresOuterRead(xact, co.managerMetadataOnFlush) val probe_initial_flags = Bits(width = nClients) probe_initial_flags := Bits(0) @@ -290,13 +290,13 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.masterMetadataOnFlush), xact_addr) + io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.managerMetadataOnFlush), xact_addr) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := xact_src io.inner.grant.bits.payload := Grant(xact_uncached, - co.getGrantType(xact, co.masterMetadataOnFlush), + co.getGrantType(xact, co.managerMetadataOnFlush), xact_client_xact_id, UInt(trackerId), UInt(0)) // Data bypassed in parent @@ -316,7 +316,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri is(s_idle) { io.inner.acquire.ready := Bool(true) val needs_outer_write = co.messageHasData(c_acq.payload) - val needs_outer_read = co.requiresOuterRead(c_acq.payload, co.masterMetadataOnFlush) + val needs_outer_read = co.requiresOuterRead(c_acq.payload, co.managerMetadataOnFlush) when( io.inner.acquire.valid ) { xact_uncached := c_acq.payload.uncached xact_a_type := c_acq.payload.a_type @@ -389,7 +389,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri when(io.outer.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { io.inner.grant.valid := Bool(true) } - when(io.inner.finish.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { + when(io.inner.finish.valid && c_ack.payload.manager_xact_id === UInt(trackerId)) { state := s_idle } } From f58f8bf3859ec09ddff6c33b488c5c23bbdd850a Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 25 Jan 2015 15:37:04 -0800 Subject: [PATCH 277/374] Make L2 data array use a single Mem --- uncore/src/main/scala/cache.scala | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 0e963f24..af3a53c7 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -214,24 +214,22 @@ class L2DataRWIO extends L2HellaCacheBundle with HasL2DataReadIO with HasL2DataW class L2DataArray extends L2HellaCacheModule { val io = new L2DataRWIO().flip - val waddr = io.write.bits.addr - val raddr = io.read.bits.addr val wmask = FillInterleaved(8, io.write.bits.wmask) - val resp = (0 until nWays).map { w => - val array = Mem(Bits(width=rowBits), nSets*refillCycles, seqRead = true) - val reg_raddr = Reg(UInt()) - when (io.write.bits.way_en(w) && io.write.valid) { - array.write(waddr, io.write.bits.data, wmask) - }.elsewhen (io.read.bits.way_en(w) && io.read.valid) { - reg_raddr := raddr - } - array(reg_raddr) + val reg_raddr = Reg(UInt()) + val array = Mem(Bits(width=rowBits), nWays*nSets*refillCycles, seqRead = true) + val waddr = Cat(OHToUInt(io.write.bits.way_en), io.write.bits.addr) + val raddr = Cat(OHToUInt(io.read.bits.way_en), io.read.bits.addr) + + when (io.write.bits.way_en.orR && io.write.valid) { + array.write(waddr, io.write.bits.data, wmask) + }.elsewhen (io.read.bits.way_en.orR && io.read.valid) { + reg_raddr := raddr } + io.resp.valid := ShiftRegister(io.read.fire(), 1) io.resp.bits.id := ShiftRegister(io.read.bits.id, 1) - io.resp.bits.data := Mux1H(ShiftRegister(io.read.bits.way_en, 1), resp) - - io.read.ready := !io.write.valid // TODO 1R/W vs 1R1W? + io.resp.bits.data := array(reg_raddr) + io.read.ready := !io.write.valid io.write.ready := Bool(true) } From 973eb43128e1294030f0527e530e0f71150d6023 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 5 Jan 2015 19:48:49 -0800 Subject: [PATCH 278/374] state machine bug on uncached write hits --- uncore/src/main/scala/cache.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index af3a53c7..468b6587 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -870,7 +870,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St xact_data(local_data_resp_cnt) := mergeData(xact, xact_data(local_data_resp_cnt), io.data.resp.bits.data) } - when(local_data_resp_done) { state := s_meta_write } + when(local_data_resp_done) { + state := Mux(co.messageHasData(xact), s_data_write, s_meta_write) + } } is(s_data_write) { io.data.write.valid := Bool(true) From 7b4e9dd137cd907760c82ce984de7d37a67ec939 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 6 Jan 2015 20:30:52 -0800 Subject: [PATCH 279/374] Block L2 transactions on the same set from proceeding in parallel --- uncore/src/main/scala/cache.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 468b6587..e7f0ce8a 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -688,8 +688,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St //TODO: Are there any races between lines with the same idx? //TODO: Allow hit under miss for stores - io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && - xact.addr(idxMSB,idxLSB) === c_acq.payload.addr(idxMSB,idxLSB) && + io.has_acquire_conflict := (co.isCoherenceConflict(xact.addr, c_acq.payload.addr) || + xact.addr(idxMSB,idxLSB) === c_acq.payload.addr(idxMSB,idxLSB)) && (state != s_idle) && !collect_cacq_data io.has_acquire_match := co.messageHasData(xact) && @@ -857,9 +857,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } } is(s_data_read) { - io.data.read.valid := Bool(true) + io.data.read.valid := (if(tlDataBeats == 1) Bool(true) + else !collect_cacq_data || (local_data_resp_cnt < cacq_data_cnt)) when(io.data.resp.valid) { - //TODO make sure cacq data is actually present before merging xact_data(local_data_resp_cnt) := mergeData(xact, xact_data(local_data_resp_cnt), io.data.resp.bits.data) } From 3aa030f9605e0dbd87be440cd747e98d0320470b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 1 Feb 2015 19:57:53 -0800 Subject: [PATCH 280/374] Support for uncached sub-block reads and writes, major TileLink and CoherencePolicy refactor. --- uncore/src/main/scala/cache.scala | 536 ++++++++---- uncore/src/main/scala/coherence.scala | 1165 ++++++++++++------------- uncore/src/main/scala/consts.scala | 6 +- uncore/src/main/scala/htif.scala | 26 +- uncore/src/main/scala/memserdes.scala | 14 +- uncore/src/main/scala/tilelink.scala | 407 ++++++--- uncore/src/main/scala/uncore.scala | 197 +++-- uncore/src/main/scala/util.scala | 6 +- 8 files changed, 1343 insertions(+), 1014 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index e7f0ce8a..6651056f 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -2,32 +2,26 @@ package uncore import Chisel._ +import scala.reflect.ClassTag case object CacheName extends Field[String] case object NSets extends Field[Int] case object NWays extends Field[Int] case object BlockOffBits extends Field[Int] case object RowBits extends Field[Int] -case object WordBits extends Field[Int] case object Replacer extends Field[() => ReplacementPolicy] +case object AmoAluOperandBits extends Field[Int] abstract trait CacheParameters extends UsesParameters { - val paddrBits = params(PAddrBits) - val vaddrBits = params(VAddrBits) - val pgIdxBits = params(PgIdxBits) val nSets = params(NSets) val blockOffBits = params(BlockOffBits) val idxBits = log2Up(nSets) val untagBits = blockOffBits + idxBits - val tagBits = paddrBits - untagBits + val tagBits = params(PAddrBits) - untagBits val nWays = params(NWays) val wayBits = log2Up(nWays) val isDM = nWays == 1 - val wordBits = params(WordBits) - val wordBytes = wordBits/8 - val wordOffBits = log2Up(wordBytes) val rowBits = params(RowBits) - val rowWords = rowBits/wordBits val rowBytes = rowBits/8 val rowOffBits = log2Up(rowBytes) } @@ -35,6 +29,79 @@ abstract trait CacheParameters extends UsesParameters { abstract class CacheBundle extends Bundle with CacheParameters abstract class CacheModule extends Module with CacheParameters +class StoreGen(typ: Bits, addr: Bits, dat: Bits) { + val byte = typ === MT_B || typ === MT_BU + val half = typ === MT_H || typ === MT_HU + val word = typ === MT_W || typ === MT_WU + def mask = + Mux(byte, Bits( 1) << addr(2,0), + Mux(half, Bits( 3) << Cat(addr(2,1), Bits(0,1)), + Mux(word, Bits( 15) << Cat(addr(2), Bits(0,2)), + Bits(255)))) + def data = + Mux(byte, Fill(8, dat( 7,0)), + Mux(half, Fill(4, dat(15,0)), + wordData)) + lazy val wordData = + Mux(word, Fill(2, dat(31,0)), + dat) +} + +class LoadGen(typ: Bits, addr: Bits, dat: Bits, zero: Bool) { + val t = new StoreGen(typ, addr, dat) + val sign = typ === MT_B || typ === MT_H || typ === MT_W || typ === MT_D + + val wordShift = Mux(addr(2), dat(63,32), dat(31,0)) + val word = Cat(Mux(t.word, Fill(32, sign && wordShift(31)), dat(63,32)), wordShift) + val halfShift = Mux(addr(1), word(31,16), word(15,0)) + val half = Cat(Mux(t.half, Fill(48, sign && halfShift(15)), word(63,16)), halfShift) + val byteShift = Mux(zero, UInt(0), Mux(addr(0), half(15,8), half(7,0))) + val byte = Cat(Mux(zero || t.byte, Fill(56, sign && byteShift(7)), half(63,8)), byteShift) +} + +class AMOALU extends CacheModule { + val operandBits = params(AmoAluOperandBits) + require(operandBits == 64) + val io = new Bundle { + val addr = Bits(INPUT, blockOffBits) + val cmd = Bits(INPUT, M_SZ) + val typ = Bits(INPUT, MT_SZ) + val lhs = Bits(INPUT, operandBits) + val rhs = Bits(INPUT, operandBits) + val out = Bits(OUTPUT, operandBits) + } + + val storegen = new StoreGen(io.typ, io.addr, io.rhs) + val rhs = storegen.wordData + + val sgned = io.cmd === M_XA_MIN || io.cmd === M_XA_MAX + val max = io.cmd === M_XA_MAX || io.cmd === M_XA_MAXU + val min = io.cmd === M_XA_MIN || io.cmd === M_XA_MINU + val word = io.typ === MT_W || io.typ === MT_WU || // Logic minimization: + io.typ === MT_B || io.typ === MT_BU + + val mask = SInt(-1,64) ^ (io.addr(2) << UInt(31)) + val adder_out = (io.lhs & mask).toUInt + (rhs & mask) + + val cmp_lhs = Mux(word && !io.addr(2), io.lhs(31), io.lhs(63)) + val cmp_rhs = Mux(word && !io.addr(2), rhs(31), rhs(63)) + val lt_lo = io.lhs(31,0) < rhs(31,0) + val lt_hi = io.lhs(63,32) < rhs(63,32) + val eq_hi = io.lhs(63,32) === rhs(63,32) + val lt = Mux(word, Mux(io.addr(2), lt_hi, lt_lo), lt_hi || eq_hi && lt_lo) + val less = Mux(cmp_lhs === cmp_rhs, lt, Mux(sgned, cmp_lhs, cmp_rhs)) + + val out = Mux(io.cmd === M_XA_ADD, adder_out, + Mux(io.cmd === M_XA_AND, io.lhs & rhs, + Mux(io.cmd === M_XA_OR, io.lhs | rhs, + Mux(io.cmd === M_XA_XOR, io.lhs ^ rhs, + Mux(Mux(less, min, max), io.lhs, + storegen.data))))) + + val wmask = FillInterleaved(8, storegen.mask) + io.out := wmask & out | ~wmask & io.lhs +} + abstract class ReplacementPolicy { def way: UInt def miss: Unit @@ -96,16 +163,35 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { io.write.ready := !rst } -abstract trait L2HellaCacheParameters extends CacheParameters - with CoherenceAgentParameters { +abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgentParameters { val idxMSB = idxBits-1 val idxLSB = 0 val refillCyclesPerBeat = params(TLDataBits)/rowBits val refillCycles = refillCyclesPerBeat*params(TLDataBeats) + require(refillCyclesPerBeat == 1) } -abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters -abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters +abstract class L2HellaCacheBundle extends TLBundle with L2HellaCacheParameters +abstract class L2HellaCacheModule extends TLModule with L2HellaCacheParameters { + def connectDataBeatCounter[S <: HasTileLinkData](inc: Bool, data: S) = { + val (cnt, cnt_done) = + Counter(inc && data.hasMultibeatData(), tlDataBeats) + val done = (inc && !data.hasMultibeatData()) || cnt_done + (cnt, done) + } + def connectOutgoingDataBeatCounter[T <: HasTileLinkData : ClassTag](in: DecoupledIO[LogicalNetworkIO[T]]) = { + connectDataBeatCounter(in.fire(), in.bits.payload) + } + def connectIncomingDataBeatCounter[T <: HasTileLinkData](in: DecoupledIO[LogicalNetworkIO[T]]) = { + connectDataBeatCounter(in.fire(), in.bits.payload)._2 + } + def connectOutgoingDataBeatCounter[T <: HasTileLinkData](in: DecoupledIO[T]) = { + connectDataBeatCounter(in.fire(), in.bits) + } + def connectIncomingDataBeatCounter[T <: HasTileLinkData](in: ValidIO[T]) = { + connectDataBeatCounter(in.valid, in.bits)._2 + } +} trait HasL2Id extends Bundle with CoherenceAgentParameters { val id = UInt(width = log2Up(nTransactors + 1)) @@ -117,6 +203,11 @@ trait HasL2InternalRequestState extends L2HellaCacheBundle { val way_en = Bits(width = nWays) } +trait HasL2Data extends HasTileLinkData { + def hasData(dummy: Int = 0) = Bool(true) + def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) +} + object L2Metadata { def apply(tag: Bits, coh: ManagerMetadata) = { val meta = new L2Metadata @@ -186,19 +277,19 @@ class L2MetadataArray extends L2HellaCacheModule { io.resp.bits.way_en := Mux(s2_tag_match, s2_tag_match_way, s2_replaced_way_en) } -class L2DataReadReq extends L2HellaCacheBundle with HasL2Id { +class L2DataReadReq extends L2HellaCacheBundle + with HasCacheBlockAddress + with HasTileLinkBeatId + with HasL2Id { val way_en = Bits(width = nWays) - val addr = Bits(width = tlAddrBits) } -class L2DataWriteReq extends L2DataReadReq { +class L2DataWriteReq extends L2DataReadReq + with HasL2Data { val wmask = Bits(width = tlWriteMaskBits) - val data = Bits(width = tlDataBits) } -class L2DataResp extends Bundle with HasL2Id with TileLinkParameters { - val data = Bits(width = tlDataBits) -} +class L2DataResp extends L2HellaCacheBundle with HasL2Id with HasL2Data trait HasL2DataReadIO extends L2HellaCacheBundle { val read = Decoupled(new L2DataReadReq) @@ -217,8 +308,8 @@ class L2DataArray extends L2HellaCacheModule { val wmask = FillInterleaved(8, io.write.bits.wmask) val reg_raddr = Reg(UInt()) val array = Mem(Bits(width=rowBits), nWays*nSets*refillCycles, seqRead = true) - val waddr = Cat(OHToUInt(io.write.bits.way_en), io.write.bits.addr) - val raddr = Cat(OHToUInt(io.read.bits.way_en), io.read.bits.addr) + val waddr = Cat(OHToUInt(io.write.bits.way_en), io.write.bits.addr_block, io.write.bits.addr_beat) + val raddr = Cat(OHToUInt(io.read.bits.way_en), io.read.bits.addr_block, io.read.bits.addr_beat) when (io.write.bits.way_en.orR && io.write.valid) { array.write(waddr, io.write.bits.data, wmask) @@ -228,6 +319,7 @@ class L2DataArray extends L2HellaCacheModule { io.resp.valid := ShiftRegister(io.read.fire(), 1) io.resp.bits.id := ShiftRegister(io.read.bits.id, 1) + io.resp.bits.addr_beat := ShiftRegister(io.read.bits.addr_beat, 1) io.resp.bits.data := array(reg_raddr) io.read.ready := !io.write.valid io.write.ready := Bool(true) @@ -261,10 +353,11 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac } // Wiring helper funcs - def doOutputArbitration[T <: Data](out: DecoupledIO[T], - ins: Seq[DecoupledIO[T]], - count: Int = 1, - lock: T => Bool = (a: T) => Bool(true)) { + def doOutputArbitration[T <: Data]( + out: DecoupledIO[T], + ins: Seq[DecoupledIO[T]], + count: Int = 1, + lock: T => Bool = (a: T) => Bool(true)) { val arb = Module(new LockingRRArbiter(out.bits.clone, ins.size, count, lock)) out <> arb.io.out arb.io.in zip ins map { case (a, in) => a <> in } @@ -331,8 +424,11 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe) :+ wb.io.inner.probe) // Wire grant reply to initiating client - def hasData(m: LogicalNetworkIO[Grant]) = co.messageHasData(m.payload) - doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant), tlDataBeats, hasData _) + doOutputArbitration( + io.inner.grant, + trackerList.map(_.io.inner.grant), + tlDataBeats, + (m: LogicalNetworkIO[Grant]) => m.payload.hasMultibeatData()) // Create an arbiter for the one memory port val outerList = trackerList.map(_.io.outer) :+ wb.io.outer @@ -344,15 +440,15 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac // Wire local memories doOutputArbitration(io.meta.read, trackerList.map(_.io.meta.read)) doOutputArbitration(io.meta.write, trackerList.map(_.io.meta.write)) - doOutputArbitration(io.data.read, trackerList.map(_.io.data.read) :+ wb.io.data.read, tlDataBeats) - doOutputArbitration(io.data.write, trackerList.map(_.io.data.write), tlDataBeats) + doOutputArbitration(io.data.read, trackerList.map(_.io.data.read) :+ wb.io.data.read) + doOutputArbitration(io.data.write, trackerList.map(_.io.data.write)) doInputRouting(io.meta.resp, trackerList.map(_.io.meta.resp)) doInputRouting(io.data.resp, trackerList.map(_.io.data.resp) :+ wb.io.data.resp) } class L2WritebackReq extends L2HellaCacheBundle with HasL2Id { - val addr = UInt(width = tlAddrBits) + val addr_block = UInt(width = tlBlockAddrBits) val coh = new ManagerMetadata val way_en = Bits(width = nWays) } @@ -373,16 +469,16 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str val has_release_match = Bool(OUTPUT) val data = new L2DataRWIO } - val c_acq = io.inner.acquire.bits - val c_rel = io.inner.release.bits - val c_gnt = io.inner.grant.bits + val cacq = io.inner.acquire.bits + val crel = io.inner.release.bits + val cgnt = io.inner.grant.bits val c_ack = io.inner.finish.bits - val m_gnt = io.outer.grant.bits + val mgnt = io.outer.grant.bits val s_idle :: s_probe :: s_data_read :: s_data_resp :: s_outer_write :: Nil = Enum(UInt(), 5) val state = Reg(init=s_idle) - val xact_addr = Reg(io.inner.acquire.bits.payload.addr.clone) + val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone) val xact_coh = Reg{ new ManagerMetadata } val xact_way_en = Reg{ Bits(width = nWays) } val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } @@ -393,30 +489,30 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str val pending_probes = Reg(init = co.dir.flush) val curr_p_id = co.dir.next(pending_probes) - val (crel_data_cnt, crel_data_done) = - Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) - val (outer_data_write_cnt, outer_data_write_done) = - Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) - val (local_data_read_cnt, local_data_read_done) = Counter(io.data.read.fire(), tlDataBeats) - val (local_data_resp_cnt, local_data_resp_done) = Counter(io.data.resp.valid, tlDataBeats) + val crel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val (macq_data_cnt, macq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire) + val (read_data_cnt, read_data_done) = Counter(io.data.read.fire(), tlDataBeats) + val resp_data_done = connectIncomingDataBeatCounter(io.data.resp) - io.has_release_match := !co.isVoluntary(c_rel.payload) && - co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && + io.has_release_match := !crel.payload.isVoluntary() && + co.isCoherenceConflict(xact_addr_block, crel.payload.addr_block) && (state === s_probe) - val next_coh_on_rel = co.managerMetadataOnRelease(c_rel.payload, xact_coh, c_rel.header.src) + val next_coh_on_rel = co.managerMetadataOnRelease(crel.payload, xact_coh, crel.header.src) io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits.payload := Bundle(UncachedWrite(xact_addr, - UInt(trackerId), - xact_data(outer_data_write_cnt)), + io.outer.acquire.bits.payload := Bundle(UncachedWriteBlock( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block, + addr_beat = macq_data_cnt, + data = xact_data(macq_data_cnt)), { case TLId => outerId }) io.outer.grant.ready := Bool(false) // Never gets mgnts io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeTypeOnVoluntaryWriteback, xact_addr) + io.inner.probe.bits.payload := Probe.onVoluntaryWriteback(xact_coh, xact_addr_block) io.inner.grant.valid := Bool(false) io.inner.acquire.ready := Bool(false) @@ -426,7 +522,8 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str io.data.read.valid := Bool(false) io.data.read.bits.id := UInt(trackerId) io.data.read.bits.way_en := xact_way_en - io.data.read.bits.addr := Cat(xact_addr, local_data_read_cnt) + io.data.read.bits.addr_block := xact_addr_block + io.data.read.bits.addr_beat := read_data_cnt io.data.write.valid := Bool(false) io.wb.req.ready := Bool(false) @@ -437,7 +534,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str is(s_idle) { io.wb.req.ready := Bool(true) when(io.wb.req.valid) { - xact_addr := io.wb.req.bits.addr + xact_addr_block := io.wb.req.bits.addr_block xact_coh := io.wb.req.bits.coh xact_way_en := io.wb.req.bits.way_en xact_id := io.wb.req.bits.id @@ -463,12 +560,12 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str when(io.inner.release.valid) { xact_coh := next_coh_on_rel // Handle released dirty data - when(co.messageHasData(c_rel.payload)) { + when(crel.payload.hasData()) { crel_had_data := Bool(true) - xact_data(crel_data_cnt) := c_rel.payload.data + xact_data(crel.payload.addr_beat) := crel.payload.data } // We don't decrement release_count until we've received all the data beats. - when(!co.messageHasData(c_rel.payload) || crel_data_done) { + when(!crel.payload.hasData() || crel_data_done) { release_count := release_count - UInt(1) } } @@ -478,16 +575,16 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str } is(s_data_read) { io.data.read.valid := Bool(true) - when(io.data.resp.valid) { xact_data(local_data_resp_cnt) := io.data.resp.bits.data } - when(local_data_read_done) { state := s_data_resp } + when(io.data.resp.valid) { xact_data(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } + when(read_data_done) { state := s_data_resp } } is(s_data_resp) { - when(io.data.resp.valid) { xact_data(local_data_resp_cnt) := io.data.resp.bits.data } - when(local_data_resp_done) { state := s_outer_write } + when(io.data.resp.valid) { xact_data(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } + when(resp_data_done) { state := s_outer_write } } is(s_outer_write) { io.outer.acquire.valid := Bool(true) - when(outer_data_write_done) { + when(macq_data_done) { io.wb.resp.valid := Bool(true) state := s_idle } @@ -508,41 +605,40 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa val wb = new L2WritebackIO } - val c_acq = io.inner.acquire.bits - val c_rel = io.inner.release.bits - val c_gnt = io.inner.grant.bits - val c_ack = io.inner.finish.bits - val m_gnt = io.outer.grant.bits - - def mergeData(acq: Acquire, old_data: UInt, new_data: UInt): UInt = { - //TODO apply acq's write mask - Mux(co.messageHasData(acq), old_data, new_data) - } + val cacq = io.inner.acquire.bits + val crel = io.inner.release.bits + val cgnt = io.inner.grant.bits + val cack = io.inner.finish.bits + val mgnt = io.outer.grant.bits } class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 7) + val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_grant :: s_ack :: Nil = Enum(UInt(), 7) val state = Reg(init=s_idle) val xact_src = Reg(io.inner.release.bits.header.src.clone) val xact_r_type = Reg(io.inner.release.bits.payload.r_type) - val xact_addr = Reg(io.inner.release.bits.payload.addr.clone) + val xact_addr_block = Reg(io.inner.release.bits.payload.addr_block.clone) + val xact_addr_beat = Reg(io.inner.release.bits.payload.addr_beat.clone) val xact_client_xact_id = Reg(io.inner.release.bits.payload.client_xact_id.clone) val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) } val xact_tag_match = Reg{ Bool() } val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } - val xact = Release(xact_r_type, xact_addr, xact_client_xact_id) + val xact = Release( + voluntary = Bool(true), + r_type = xact_r_type, + client_xact_id = xact_client_xact_id, + addr_block = xact_addr_block) - val collect_inner_data = Reg(init=Bool(false)) - val (inner_data_cnt, inner_data_done) = - Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) - val (local_data_cnt, local_data_done) = - Counter(io.data.write.fire(), tlDataBeats) + val collect_crel_data = Reg(init=Bool(false)) + val crel_data_valid = Reg(init=Bits(0, width = tlDataBeats)) + val crel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val (write_data_cnt, write_data_done) = connectOutgoingDataBeatCounter(io.data.write) io.has_acquire_conflict := Bool(false) io.has_acquire_match := Bool(false) - io.has_release_match := co.isVoluntary(c_rel.payload) + io.has_release_match := crel.payload.isVoluntary() io.outer.grant.ready := Bool(false) io.outer.acquire.valid := Bool(false) @@ -554,50 +650,51 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := Grant(Bool(false), - co.getGrantTypeOnVoluntaryWriteback(xact_meta.coh), - xact_client_xact_id, - UInt(trackerId)) + io.inner.grant.bits.payload := xact.makeGrant(UInt(trackerId), xact_meta.coh) io.data.read.valid := Bool(false) io.data.write.valid := Bool(false) io.data.write.bits.id := UInt(trackerId) io.data.write.bits.way_en := xact_way_en - io.data.write.bits.addr := Cat(xact_addr, local_data_cnt) + io.data.write.bits.addr_block := xact_addr_block + io.data.write.bits.addr_beat := write_data_cnt io.data.write.bits.wmask := SInt(-1) - io.data.write.bits.data := xact_data(local_data_cnt) + io.data.write.bits.data := xact_data(write_data_cnt) io.meta.read.valid := Bool(false) io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact_addr(idxMSB,idxLSB) - io.meta.read.bits.tag := xact_addr >> UInt(idxBits) + io.meta.read.bits.idx := xact_addr_block(idxMSB,idxLSB) + io.meta.read.bits.tag := xact_addr_block >> UInt(idxBits) io.meta.write.valid := Bool(false) io.meta.write.bits.id := UInt(trackerId) - io.meta.write.bits.idx := xact_addr(idxMSB,idxLSB) + io.meta.write.bits.idx := xact_addr_block(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en - io.meta.write.bits.data.tag := xact_addr >> UInt(idxBits) + io.meta.write.bits.data.tag := xact_addr_block >> UInt(idxBits) io.meta.write.bits.data.coh := co.managerMetadataOnRelease(xact, xact_meta.coh, xact_src) io.wb.req.valid := Bool(false) - when(collect_inner_data) { + when(collect_crel_data) { io.inner.release.ready := Bool(true) when(io.inner.release.valid) { - xact_data(inner_data_cnt) := c_rel.payload.data + xact_data(crel.payload.addr_beat) := crel.payload.data + crel_data_valid(crel.payload.addr_beat) := Bool(true) } - when(inner_data_done) { collect_inner_data := Bool(false) } + when(crel_data_done) { collect_crel_data := Bool(false) } } switch (state) { is(s_idle) { io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { - xact_src := c_rel.header.src - xact_r_type := c_rel.payload.r_type - xact_addr := c_rel.payload.addr - xact_client_xact_id := c_rel.payload.client_xact_id - xact_data(UInt(0)) := c_rel.payload.data - collect_inner_data := co.messageHasData(c_rel.payload) + xact_src := crel.header.src + xact_r_type := crel.payload.r_type + xact_addr_block := crel.payload.addr_block + xact_addr_beat := crel.payload.addr_beat + xact_client_xact_id := crel.payload.client_xact_id + xact_data(UInt(0)) := crel.payload.data + collect_crel_data := crel.payload.hasMultibeatData() + crel_data_valid := Bits(1) state := s_meta_read } } @@ -611,27 +708,30 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou xact_meta := io.meta.resp.bits.meta xact_way_en := io.meta.resp.bits.way_en state := Mux(io.meta.resp.bits.tag_match, - Mux(co.messageHasData(xact), s_data_write, s_meta_write), - s_grant) + Mux(xact.hasData(), s_data_write, s_meta_write), + Mux(xact.requiresAck(), s_grant, s_idle)) } } is(s_data_write) { - io.data.write.valid := (if(tlDataBeats == 1) Bool(true) - else !collect_inner_data || (local_data_cnt < inner_data_cnt)) - when(local_data_done) { state := s_meta_write } + io.data.write.valid := !collect_crel_data || crel_data_valid(write_data_cnt) + when(write_data_done) { state := s_meta_write } } is(s_meta_write) { io.meta.write.valid := Bool(true) - when(io.meta.write.ready) { state := s_grant } + when(io.meta.write.ready) { + state := Mux(xact.requiresAck(), s_grant, s_idle) // Need a Grant.voluntaryAck? + } } is(s_grant) { io.inner.grant.valid := Bool(true) when(io.inner.grant.ready) { - state := Mux(co.requiresAckForGrant(c_gnt.payload), - s_busy, s_idle) + state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) } } - is(s_busy) { + is(s_ack) { + // TODO: This state is unnecessary if no client will ever issue the + // pending Acquire that caused this writeback until it receives the + // Grant.voluntaryAck for this writeback io.inner.finish.ready := Bool(true) when(io.inner.finish.valid) { state := s_idle } } @@ -640,72 +740,109 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_probe :: s_outer_read :: s_outer_resp :: s_data_read :: s_data_resp :: s_data_write :: s_meta_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 14) + val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_probe :: s_outer_read :: s_outer_resp :: s_data_read :: s_data_resp :: s_data_write :: s_meta_write :: s_grant :: s_ack :: Nil = Enum(UInt(), 14) val state = Reg(init=s_idle) val xact_src = Reg(io.inner.acquire.bits.header.src.clone) val xact_uncached = Reg(io.inner.acquire.bits.payload.uncached.clone) val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone) - val xact_addr = Reg(io.inner.acquire.bits.payload.addr.clone) + val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone) + val xact_addr_beat = Reg(io.inner.acquire.bits.payload.addr_beat.clone) val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone) val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone) val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } val xact_tag_match = Reg{ Bool() } val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } - val xact = Acquire(xact_uncached, xact_a_type, xact_addr, xact_client_xact_id, UInt(0), xact_subblock) + val xact = Acquire( + uncached = xact_uncached, + a_type = xact_a_type, + client_xact_id = xact_client_xact_id, + addr_block = xact_addr_block, + addr_beat = xact_addr_beat, + data = UInt(0), + subblock = xact_subblock) + val collect_cacq_data = Reg(init=Bool(false)) + val cacq_data_valid = Reg(init=Bits(0, width = tlDataBeats)) val crel_had_data = Reg(init = Bool(false)) val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) val pending_probes = Reg(init = UInt(0, width = nCoherentClients)) val curr_p_id = co.dir.next(pending_probes) val full_sharers = co.dir.full(io.meta.resp.bits.meta.coh.sharers) - val mask_self = Mux(co.requiresSelfProbe(xact), + val mask_self = Mux(xact.requiresSelfProbe(), full_sharers | (UInt(1) << xact_src), full_sharers & ~UInt(UInt(1) << xact_src, width = nClients)) val mask_incoherent = mask_self & ~io.tile_incoherent - val collect_cacq_data = Reg(init=Bool(false)) - //TODO: zero width wires - val (cacq_data_cnt, cacq_data_done) = - Counter(io.inner.acquire.fire() && co.messageHasData(io.inner.acquire.bits.payload), tlDataBeats) - val (crel_data_cnt, crel_data_done) = - Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) - val (cgnt_data_cnt, cgnt_data_done) = - Counter(io.inner.grant.fire() && co.messageHasData(io.inner.grant.bits.payload), tlDataBeats) - val (outer_data_write_cnt, outer_data_write_done) = - Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) - val (outer_data_resp_cnt, outer_data_resp_done) = - Counter(io.outer.grant.fire() && co.messageHasData(io.outer.grant.bits.payload), tlDataBeats) - val (local_data_read_cnt, local_data_read_done) = Counter(io.data.read.fire(), tlDataBeats) - val (local_data_write_cnt, local_data_write_done) = Counter(io.data.write.fire(), tlDataBeats) - val (local_data_resp_cnt, local_data_resp_done) = Counter(io.data.resp.valid, tlDataBeats) + val cacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) + val crel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val (macq_data_cnt, macq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire) + val mgnt_data_done = connectIncomingDataBeatCounter(io.outer.grant) + val cgnt_data_cnt = Reg(init = UInt(0, width = tlBeatAddrBits+1)) + val cgnt_data_max = Reg(init = UInt(0, width = tlBeatAddrBits+1)) + val read_data_cnt = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) + val read_data_max = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) + val write_data_cnt = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) + val write_data_max = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) + val resp_data_cnt = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) + val resp_data_max = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) - val needs_writeback = !xact_tag_match && co.needsWriteback(xact_meta.coh) + val needs_writeback = !xact_tag_match && co.isValid(xact_meta.coh) // TODO: dirty bit val is_hit = xact_tag_match && co.isHit(xact, xact_meta.coh) val needs_probes = co.requiresProbes(xact, xact_meta.coh) - //TODO: uncached does or does not allocate + //val do_allocate = !xact_uncached || xact.allocate() + + val amoalu = Module(new AMOALU) + amoalu.io.addr := xact.addr() + amoalu.io.cmd := xact.op_code() + amoalu.io.typ := xact.op_size() + amoalu.io.lhs := io.data.resp.bits.data //default + amoalu.io.rhs := xact.data(0) // default + + def mergeData[T <: HasTileLinkData](buffer: Vec[UInt], incoming: T) { + val old_data = incoming.data + val new_data = buffer(incoming.addr_beat) + amoalu.io.lhs := old_data + amoalu.io.rhs := new_data + val wmask = FillInterleaved(8, xact.write_mask()) + buffer(incoming.addr_beat) := + Mux(xact.is(Acquire.uncachedAtomic), amoalu.io.out, + Mux(xact.is(Acquire.uncachedWriteBlock) || xact.is(Acquire.uncachedWrite), + wmask & new_data | ~wmask & old_data, old_data)) + } - //TODO: Are there any races between lines with the same idx? //TODO: Allow hit under miss for stores - io.has_acquire_conflict := (co.isCoherenceConflict(xact.addr, c_acq.payload.addr) || - xact.addr(idxMSB,idxLSB) === c_acq.payload.addr(idxMSB,idxLSB)) && + io.has_acquire_conflict := (co.isCoherenceConflict(xact.addr_block, cacq.payload.addr_block) || + xact.addr_block(idxMSB,idxLSB) === cacq.payload.addr_block(idxMSB,idxLSB)) && (state != s_idle) && !collect_cacq_data - io.has_acquire_match := co.messageHasData(xact) && - (xact.addr === c_acq.payload.addr) && + io.has_acquire_match := xact.hasMultibeatData() && + (xact.addr_block === cacq.payload.addr_block) && collect_cacq_data - io.has_release_match := !co.isVoluntary(c_rel.payload) && - (xact.addr === c_rel.payload.addr) && + io.has_release_match := !crel.payload.isVoluntary() && + (xact.addr_block === crel.payload.addr_block) && (state === s_probe) - val next_coh_on_rel = co.managerMetadataOnRelease(c_rel.payload, xact_meta.coh, c_rel.header.src) - val next_coh_on_gnt = co.managerMetadataOnGrant(c_gnt.payload, xact_meta.coh, - c_gnt.header.dst) + val next_coh_on_rel = co.managerMetadataOnRelease( + incoming = crel.payload, + meta = xact_meta.coh, + src = crel.header.src) + val next_coh_on_gnt = co.managerMetadataOnGrant( + outgoing = cgnt.payload, + meta = xact_meta.coh, + dst = cgnt.header.dst) - val outer_write = Bundle(UncachedWrite(xact_addr, UInt(trackerId), xact_data(outer_data_write_cnt)), - { case TLId => outerId }) - val outer_read = Bundle(UncachedRead( xact_addr, UInt(trackerId)), { case TLId => outerId }) + val outer_write = Bundle(UncachedWriteBlock( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block, + addr_beat = macq_data_cnt, + data = xact_data(macq_data_cnt)), + { case TLId => outerId }) + val outer_read = Bundle(UncachedReadBlock( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block), + { case TLId => outerId }) io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.payload := outer_read //default @@ -714,15 +851,16 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact, xact_meta.coh), xact_addr) + io.inner.probe.bits.payload := xact.makeProbe(xact_meta.coh) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := Grant(xact_uncached, co.getGrantType(xact, xact_meta.coh), - xact_client_xact_id, - UInt(trackerId), - xact_data(cgnt_data_cnt)) + io.inner.grant.bits.payload := xact.makeGrant( + manager_xact_id = UInt(trackerId), + meta = xact_meta.coh, + addr_beat = cgnt_data_cnt, + data = xact_data(cgnt_data_cnt)) io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) @@ -731,26 +869,28 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.data.read.valid := Bool(false) io.data.read.bits.id := UInt(trackerId) io.data.read.bits.way_en := xact_way_en - io.data.read.bits.addr := Cat(xact_addr, local_data_read_cnt) + io.data.read.bits.addr_block := xact_addr_block + io.data.read.bits.addr_beat := read_data_cnt io.data.write.valid := Bool(false) io.data.write.bits.id := UInt(trackerId) io.data.write.bits.way_en := xact_way_en - io.data.write.bits.addr := Cat(xact_addr, local_data_write_cnt) + io.data.write.bits.addr_block := xact_addr_block + io.data.write.bits.addr_beat := write_data_cnt io.data.write.bits.wmask := SInt(-1) - io.data.write.bits.data := xact_data(local_data_write_cnt) + io.data.write.bits.data := xact_data(write_data_cnt) io.meta.read.valid := Bool(false) io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact_addr(idxMSB,idxLSB) - io.meta.read.bits.tag := xact_addr >> UInt(idxBits) + io.meta.read.bits.idx := xact_addr_block(idxMSB,idxLSB) + io.meta.read.bits.tag := xact_addr_block >> UInt(idxBits) io.meta.write.valid := Bool(false) io.meta.write.bits.id := UInt(trackerId) - io.meta.write.bits.idx := xact_addr(idxMSB,idxLSB) + io.meta.write.bits.idx := xact_addr_block(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en - io.meta.write.bits.data.tag := xact_addr >> UInt(idxBits) + io.meta.write.bits.data.tag := xact_addr_block >> UInt(idxBits) io.meta.write.bits.data.coh := next_coh_on_gnt io.wb.req.valid := Bool(false) - io.wb.req.bits.addr := Cat(xact_meta.tag, xact_addr(idxMSB,idxLSB)) + io.wb.req.bits.addr_block := Cat(xact_meta.tag, xact_addr_block(idxMSB,idxLSB)) io.wb.req.bits.coh := xact_meta.coh io.wb.req.bits.way_en := xact_way_en io.wb.req.bits.id := UInt(trackerId) @@ -758,7 +898,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St when(collect_cacq_data) { io.inner.acquire.ready := Bool(true) when(io.inner.acquire.valid) { - xact_data(cacq_data_cnt) := c_acq.payload.data + xact_data(cacq.payload.addr_beat) := cacq.payload.data + cacq_data_valid(cacq.payload.addr_beat) := Bool(true) } when(cacq_data_done) { collect_cacq_data := Bool(false) } } @@ -767,14 +908,15 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St is(s_idle) { io.inner.acquire.ready := Bool(true) when( io.inner.acquire.valid ) { - xact_uncached := c_acq.payload.uncached - xact_a_type := c_acq.payload.a_type - xact_addr := c_acq.payload.addr - xact_client_xact_id := c_acq.payload.client_xact_id - xact_data(UInt(0)) := c_acq.payload.data - xact_subblock := c_acq.payload.subblock - xact_src := c_acq.header.src - collect_cacq_data := co.messageHasData(c_acq.payload) + xact_uncached := cacq.payload.uncached + xact_a_type := cacq.payload.a_type + xact_addr_block := cacq.payload.addr_block + xact_addr_beat := cacq.payload.addr_beat + xact_client_xact_id := cacq.payload.client_xact_id + xact_data(UInt(0)) := cacq.payload.data + xact_subblock := cacq.payload.subblock + xact_src := cacq.header.src + collect_cacq_data := cacq.payload.hasMultibeatData() state := s_meta_read } } @@ -789,9 +931,20 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St xact_way_en := io.meta.resp.bits.way_en val coh = io.meta.resp.bits.meta.coh val _tag_match = io.meta.resp.bits.tag_match - val _needs_writeback = !_tag_match && co.needsWriteback(coh) + val _needs_writeback = !_tag_match && co.isValid(coh) //TODO: dirty bit val _needs_probes = _tag_match && co.requiresProbes(xact, coh) val _is_hit = _tag_match && co.isHit(xact, coh) + val full_block = !xact.uncached || + xact.is(Acquire.uncachedReadBlock) || + xact.is(Acquire.uncachedWriteBlock) + read_data_cnt := Mux(full_block, UInt(0), xact_addr_beat) + read_data_max := Mux(full_block, UInt(refillCycles-1), xact_addr_beat) + write_data_cnt := Mux(full_block || !_is_hit, UInt(0), xact_addr_beat) + write_data_max := Mux(full_block || !_is_hit, UInt(refillCycles-1), xact_addr_beat) + resp_data_cnt := Mux(full_block, UInt(0), xact_addr_beat) + resp_data_max := Mux(full_block, UInt(refillCycles-1), xact_addr_beat) + cgnt_data_cnt := Mux(full_block, UInt(0), xact_addr_beat) + cgnt_data_max := Mux(full_block, UInt(tlDataBeats-1), xact_addr_beat) when(_needs_probes) { pending_probes := mask_incoherent(nCoherentClients-1,0) release_count := co.dir.count(mask_incoherent) @@ -816,19 +969,18 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St pending_probes := co.dir.pop(pending_probes, curr_p_id) } // Handle releases, which may have data being written back + //TODO: make sure cacq data is actually present before accpeting + // release data to merge! io.inner.release.ready := Bool(true) when(io.inner.release.valid) { xact_meta.coh := next_coh_on_rel // Handle released dirty data - when(co.messageHasData(c_rel.payload)) { + when(crel.payload.hasData()) { crel_had_data := Bool(true) - //TODO make sure cacq data is actually present before merging - xact_data(crel_data_cnt) := mergeData(xact, - xact_data(crel_data_cnt), - c_rel.payload.data) + mergeData(xact_data, crel.payload) } // We don't decrement release_count until we've received all the data beats. - when(!co.messageHasData(c_rel.payload) || crel_data_done) { + when(!crel.payload.hasMultibeatData() || crel_data_done) { release_count := release_count - UInt(1) } } @@ -847,37 +999,39 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.outer.grant.ready := Bool(true) when(io.outer.grant.valid) { //TODO make sure cacq data is actually present before merging - xact_data(outer_data_resp_cnt) := mergeData(xact, xact_data(outer_data_resp_cnt), - io.outer.grant.bits.payload.data) - //TODO: set pending client state in xact_meta.coh - when(outer_data_resp_done) { - state := Mux(co.messageHasData(io.outer.grant.bits.payload), - s_data_write, s_data_read) + mergeData(xact_data, mgnt.payload) + when(mgnt_data_done) { + state := Mux(mgnt.payload.hasData(), s_data_write, s_data_read) } } } is(s_data_read) { - io.data.read.valid := (if(tlDataBeats == 1) Bool(true) - else !collect_cacq_data || (local_data_resp_cnt < cacq_data_cnt)) + io.data.read.valid := !collect_cacq_data || cacq_data_valid(read_data_cnt) when(io.data.resp.valid) { - xact_data(local_data_resp_cnt) := mergeData(xact, xact_data(local_data_resp_cnt), - io.data.resp.bits.data) + mergeData(xact_data, io.data.resp.bits) + resp_data_cnt := resp_data_cnt + UInt(1) + } + when(io.data.read.ready) { + read_data_cnt := read_data_cnt + UInt(1) + when(read_data_cnt === read_data_max) { state := s_data_resp } } - when(local_data_read_done) { state := s_data_resp } } is(s_data_resp) { when(io.data.resp.valid) { - xact_data(local_data_resp_cnt) := mergeData(xact, xact_data(local_data_resp_cnt), - io.data.resp.bits.data) + mergeData(xact_data, io.data.resp.bits) + resp_data_cnt := resp_data_cnt + UInt(1) } - when(local_data_resp_done) { - state := Mux(co.messageHasData(xact), s_data_write, s_meta_write) + when(resp_data_cnt === resp_data_max) { + state := Mux(xact.hasData(), s_data_write, s_meta_write) } } is(s_data_write) { io.data.write.valid := Bool(true) - when(local_data_write_done) { - state := s_meta_write + when(io.data.write.ready) { + write_data_cnt := write_data_cnt + UInt(1) + when(write_data_cnt === write_data_max) { + state := s_meta_write + } } } is(s_meta_write) { @@ -886,12 +1040,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } is(s_grant) { io.inner.grant.valid := Bool(true) - when(!co.messageHasData(c_gnt.payload) || cgnt_data_done) { - state := Mux(co.requiresAckForGrant(c_gnt.payload), - s_busy, s_idle) + when(io.inner.grant.ready) { + cgnt_data_cnt := cgnt_data_cnt + UInt(1) + when(cgnt_data_cnt === cgnt_data_max) { + state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) + } } } - is(s_busy) { + is(s_ack) { io.inner.finish.ready := Bool(true) when(io.inner.finish.valid) { state := s_idle } } diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 08d7cf20..6367b44d 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -3,7 +3,6 @@ package uncore import Chisel._ - // Classes to represent coherence information in clients and managers abstract class CoherenceMetadata extends Bundle with CoherenceAgentParameters @@ -14,9 +13,9 @@ class ClientMetadata extends CoherenceMetadata { } object ClientMetadata { def apply(state: UInt) = { - val m = new ClientMetadata - m.state := state - m + val meta = new ClientMetadata + meta.state := state + meta } } @@ -27,10 +26,10 @@ class ManagerMetadata extends CoherenceMetadata { } object ManagerMetadata { def apply(state: UInt, sharers: UInt) = { - val m = new ManagerMetadata - m.state := state - m.sharers := sharers - m + val meta = new ManagerMetadata + meta.state := state + meta.sharers := sharers + meta } def apply(state: UInt): ManagerMetadata = apply(state, new ManagerMetadata().co.dir.flush) } @@ -85,56 +84,59 @@ abstract class CoherencePolicy(val dir: DirectoryRepresentation) { def releaseTypeWidth = log2Up(nReleaseTypes) def grantTypeWidth = log2Up(nGrantTypes) - def isHit (cmd: UInt, m: ClientMetadata): Bool - def isValid (m: ClientMetadata): Bool - def isHit (incoming: Acquire, m: ManagerMetadata): Bool - def isValid (m: ManagerMetadata): Bool + val clientStatesWithReadPermission: Vec[UInt] + val clientStatesWithWritePermission: Vec[UInt] + val clientStatesWithDirtyData: Vec[UInt] + val acquireTypesWithData = Nil // Only built-in Acquire types have data for now + val releaseTypesWithData: Vec[UInt] + val grantTypesWithData: Vec[UInt] + + def isValid(meta: ClientMetadata): Bool + def isValid(meta: ManagerMetadata): Bool + + def isHit(cmd: UInt, meta: ClientMetadata): Bool = { + Mux(isWriteIntent(cmd), + clientStatesWithWritePermission.contains(meta.state), + clientStatesWithReadPermission.contains(meta.state)) + } + //TODO: Use outer protocol's clientState instead, remove this function: + def isHit(incoming: Acquire, meta: ManagerMetadata) = isValid(meta) def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool - def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool - def needsWriteback(m: ClientMetadata): Bool - def needsWriteback(m: ManagerMetadata): Bool + //TODO: Assumes all cache ctrl ops writeback dirty data, and + // doesn't issue transaction when e.g. downgrading Exclusive to Shared: + def needsTransactionOnCacheControl(cmd: UInt, meta: ClientMetadata): Bool = + clientStatesWithDirtyData.contains(meta.state) + def needsWriteback(meta: ClientMetadata): Bool = + needsTransactionOnCacheControl(M_FLUSH, meta) - def clientMetadataOnHit(cmd: UInt, m: ClientMetadata): ClientMetadata - def clientMetadataOnCacheControl(cmd: UInt): ClientMetadata - def clientMetadataOnFlush: ClientMetadata def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire): ClientMetadata - def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata): ClientMetadata - def managerMetadataOnFlush: ManagerMetadata - def managerMetadataOnRelease(incoming: Release, m: ManagerMetadata, src: UInt): ManagerMetadata - def managerMetadataOnGrant(outgoing: Grant, m: ManagerMetadata, dst: UInt): ManagerMetadata + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata): ClientMetadata + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata): ClientMetadata + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata): ClientMetadata + def clientMetadataOnFlush: ClientMetadata - def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt - def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt - def getProbeType(a: Acquire, m: ManagerMetadata): UInt - def getProbeTypeOnVoluntaryWriteback: UInt - def getReleaseTypeOnCacheControl(cmd: UInt): UInt - def getReleaseTypeOnVoluntaryWriteback(): UInt - def getReleaseTypeOnProbe(p: Probe, m: ClientMetadata): UInt - def getGrantType(a: Acquire, m: ManagerMetadata): UInt - def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt + def managerMetadataOnRelease(incoming: Release, meta: ManagerMetadata, src: UInt): ManagerMetadata + def managerMetadataOnGrant(outgoing: Grant, meta: ManagerMetadata, dst: UInt): ManagerMetadata + def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata): ManagerMetadata + def managerMetadataOnFlush: ManagerMetadata - def messageHasData(rel: TileLinkChannel): Bool - def messageUpdatesDataArray(g: Grant): Bool - def isVoluntary(rel: Release): Bool - def isVoluntary(gnt: Grant): Bool - def requiresOuterRead(acq: Acquire, m: ManagerMetadata): Bool - def requiresOuterWrite(acq: Acquire, m: ManagerMetadata): Bool - def requiresSelfProbe(a: Acquire): Bool - def requiresProbes(a: Acquire, m: ManagerMetadata): Bool - def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata): Bool - def requiresAckForGrant(g: Grant): Bool - def requiresAckForRelease(r: Release): Bool + def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt + def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt + def getReleaseType(p: Probe, meta: ClientMetadata): UInt + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt + + def getGrantType(acq: Acquire, meta: ManagerMetadata): UInt + def getProbeType(acq: Acquire, meta: ManagerMetadata): UInt + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt + + def requiresOuterRead(acq: Acquire, meta: ManagerMetadata): Bool + def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata): Bool + def requiresProbes(acq: Acquire, meta: ManagerMetadata): Bool + def requiresProbes(cmd: UInt, meta: ManagerMetadata): Bool + def requiresProbesOnVoluntaryWriteback(meta: ManagerMetadata): Bool = requiresProbes(M_FLUSH, meta) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool - - def getGrantTypeForUncached(a: Acquire, m: ManagerMetadata): UInt = { - MuxLookup(a.a_type, Grant.uncachedRead, Array( - Acquire.uncachedRead -> Grant.uncachedRead, - Acquire.uncachedWrite -> Grant.uncachedWrite, - Acquire.uncachedAtomic -> Grant.uncachedAtomic - )) - } } class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { @@ -142,123 +144,105 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nManagerStates = 2 def nAcquireTypes = 1 def nProbeTypes = 2 - def nReleaseTypes = 5 - def nGrantTypes = 2 + def nReleaseTypes = 4 + def nGrantTypes = 1 val clientInvalid :: clientValid :: Nil = Enum(UInt(), nClientStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) + val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) - val acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) + val acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadExclusive :: Nil = Enum(UInt(), nGrantTypes) + val releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantExclusive :: Nil = Enum(UInt(), nGrantTypes) - val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseCopyData) - val hasDataGrantTypeVec = Vec(grantReadExclusive) + val clientStatesWithReadPermission = Vec(clientValid) + val clientStatesWithWritePermission = Vec(clientValid) + val clientStatesWithDirtyData = Vec(clientValid) + val releaseTypesWithData = Vec(releaseInvalidateData, releaseCopyData) + val grantTypesWithData = Vec(grantExclusive) - def isHit (cmd: UInt, m: ClientMetadata): Bool = isValid(m) - def isValid (m: ClientMetadata): Bool = m.state != clientInvalid - def isHit (incoming: Acquire, m: ManagerMetadata): Bool = isValid(m) - def isValid (m: ManagerMetadata): Bool = m.state != masterInvalid + def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid + def isValid (meta: ManagerMetadata): Bool = meta.state != managerInvalid - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = (outstanding.a_type != acquireReadExclusive) - def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { - MuxLookup(cmd, (m.state === clientValid), Array( - M_INV -> (m.state === clientValid), - M_CLN -> (m.state === clientValid) - )) - } - def needsWriteback (m: ClientMetadata): Bool = { - needsTransactionOnCacheControl(M_INV, m) - } - def needsWriteback(m: ManagerMetadata) = isValid(m) + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = + (outstanding.a_type != acquireExclusive) - def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = m - def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( - MuxLookup(cmd, clientInvalid, Array( - M_INV -> clientInvalid, - M_CLN -> clientValid - ))) - def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = meta + + def clientMetadataOnFlush = ClientMetadata(clientInvalid) + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(cmd === M_FLUSH, clientInvalid, meta.state)) + + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata(Mux(incoming.uncached, clientInvalid, clientValid)) - def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( - MuxLookup(incoming.p_type, m.state, Array( - probeInvalidate -> clientInvalid, - probeCopy -> m.state - ))) - def managerMetadataOnFlush = ManagerMetadata(masterInvalid) - def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(masterValid, dir.pop(m.sharers, src)) - MuxBundle(m, Array( - r.is(releaseVoluntaryInvalidateData) -> next, + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata(Mux(incoming.p_type === probeInvalidate, + clientInvalid, meta.state)) + + def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(managerValid, dir.pop(meta.sharers, src)) + MuxBundle(meta, Array( r.is(releaseInvalidateData) -> next, r.is(releaseInvalidateAck) -> next )) } - def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { - val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = ManagerMetadata(masterValid, m.sharers) - Mux(g.uncached, uncached, cached) + + def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = + Mux(g.uncached, + ManagerMetadata(managerValid, meta.sharers), + ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) + + def managerMetadataOnFlush = ManagerMetadata(managerInvalid) + + def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = + ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) + + def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = acquireExclusive + + def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = acquireExclusive + + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) + MuxLookup(cmd, releaseCopyAck, Array( + M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), + M_PRODUCE -> Mux(dirty, releaseCopyData, releaseCopyAck), + M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) } - def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = acquireReadExclusive - def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = acquireReadExclusive - def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { - val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> releaseInvalidateData, - probeCopy -> releaseCopyData - )) - val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( - probeInvalidate -> releaseInvalidateAck, - probeCopy -> releaseCopyAck - )) - Mux(needsWriteback(m), with_data, without_data) - } - - def messageHasData(msg: TileLinkChannel) = msg match { - case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) - case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) - case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) - case _ => Bool(false) - } - def messageUpdatesDataArray(g: Grant): Bool = { - Mux(g.uncached, Bool(false), - (g.g_type === grantReadExclusive)) - } + def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = + MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> getReleaseType(M_FLUSH, meta), + probeCopy -> getReleaseType(M_CLEAN, meta))) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: ManagerMetadata): UInt = - Mux(a.uncached, getGrantTypeForUncached(a, m), grantReadExclusive) + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.uncached, Grant.getGrantTypeForUncached(a), grantExclusive) - def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck - - def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedReadBlock -> probeCopy, + Acquire.uncachedWriteBlock -> probeInvalidate, Acquire.uncachedRead -> probeCopy, Acquire.uncachedWrite -> probeInvalidate, Acquire.uncachedAtomic -> probeInvalidate )), probeInvalidate) - } - def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate - def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = + MuxLookup(cmd, probeCopy, Array( + M_FLUSH -> probeInvalidate)) + + def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) - def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = + + def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) - def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck - def requiresAckForRelease(r: Release) = Bool(false) - def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) - def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) + def requiresProbes(a: Acquire, meta: ManagerMetadata) = !dir.none(meta.sharers) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) } class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { @@ -266,137 +250,123 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nManagerStates = 2 def nAcquireTypes = 2 def nProbeTypes = 3 - def nReleaseTypes = 7 - def nGrantTypes = 2 + def nReleaseTypes = 6 + def nGrantTypes = 1 val clientInvalid :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) + val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) - val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: Nil = Enum(UInt(), nAcquireTypes) + val acquireExclusiveClean :: acquireExclusiveDirty :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadExclusive :: Nil = Enum(UInt(), nGrantTypes) + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantExclusive :: Nil = Enum(UInt(), nGrantTypes) - val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeVec = Vec(grantReadExclusive) + val clientStatesWithReadPermission = Vec(clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty) + val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val grantTypesWithData = Vec(grantExclusive) - def isHit (cmd: UInt, m: ClientMetadata) = isValid(m) - def isValid (m: ClientMetadata) = m.state != clientInvalid - def isHit (incoming: Acquire, m: ManagerMetadata) = isValid(m) - def isValid (m: ManagerMetadata) = m.state != masterInvalid + def isValid (meta: ClientMetadata) = meta.state != clientInvalid + def isValid (meta: ManagerMetadata) = meta.state != managerInvalid - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = (isRead(cmd) && outstanding.uncached) || - (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusiveDirty)) - } - def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { - MuxLookup(cmd, (m.state === clientExclusiveDirty), Array( - M_INV -> (m.state === clientExclusiveDirty), - M_CLN -> (m.state === clientExclusiveDirty) - )) - } - def needsWriteback (m: ClientMetadata): Bool = { - needsTransactionOnCacheControl(M_INV, m) - } - def needsWriteback(m: ManagerMetadata) = isValid(m) + (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusiveDirty)) - def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = - ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state)) - - def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( - MuxLookup(cmd, clientInvalid, Array( - M_INV -> clientInvalid, - M_CLN -> clientExclusiveClean - ))) - def clientMetadataOnFlush() = clientMetadataOnCacheControl(M_INV) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - Mux(incoming.uncached, clientInvalid, - Mux(outstanding.a_type === acquireReadExclusiveDirty, clientExclusiveDirty, - clientExclusiveClean))) - def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( - MuxLookup(incoming.p_type, m.state, Array( - probeInvalidate -> clientInvalid, - probeDowngrade -> clientExclusiveClean, - probeCopy -> m.state - ))) - def managerMetadataOnFlush = ManagerMetadata(masterInvalid) - def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(masterValid, dir.pop(m.sharers,src)) - MuxBundle(m, Array( - r.is(releaseVoluntaryInvalidateData) -> next, + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) + + def clientMetadataOnFlush = ClientMetadata(clientInvalid) + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(cmd, meta.state, Array( + M_FLUSH -> clientInvalid, + M_CLEAN -> Mux(meta.state === clientExclusiveDirty, clientExclusiveClean, meta.state)))) + + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = + ClientMetadata( + Mux(incoming.uncached, clientInvalid, + Mux(outstanding.a_type === acquireExclusiveDirty, clientExclusiveDirty, + clientExclusiveClean))) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(incoming.p_type, meta.state, Array( + probeInvalidate -> clientInvalid, + probeDowngrade -> clientExclusiveClean, + probeCopy -> meta.state))) + + def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(managerValid, dir.pop(meta.sharers,src)) + MuxBundle(meta, Array( r.is(releaseInvalidateData) -> next, r.is(releaseInvalidateAck) -> next )) } - def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { - val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = ManagerMetadata(masterValid, m.sharers) - Mux(g.uncached, uncached, cached) + + def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = + Mux(g.uncached, + ManagerMetadata(managerValid, meta.sharers), + ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) + + def managerMetadataOnFlush = ManagerMetadata(managerInvalid) + + def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = + ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) + + def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = + Mux(isWriteIntent(cmd), acquireExclusiveDirty, acquireExclusiveClean) + + def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = + Mux(isWriteIntent(cmd), acquireExclusiveDirty, outstanding.a_type) + + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) + MuxLookup(cmd, releaseCopyAck, Array( + M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), + M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), + M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) } - def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { - Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, acquireReadExclusiveClean) - } - def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = { - Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, outstanding.a_type) - } - def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { - val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> releaseInvalidateData, - probeDowngrade -> releaseDowngradeData, - probeCopy -> releaseCopyData - )) - val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( - probeInvalidate -> releaseInvalidateAck, - probeDowngrade -> releaseDowngradeAck, - probeCopy -> releaseCopyAck - )) - Mux(needsWriteback(m), with_data, without_data) - } - - def messageHasData(msg: TileLinkChannel) = msg match { - case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) - case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) - case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) - case _ => Bool(false) - } - def messageUpdatesDataArray(g: Grant): Bool = { - Mux(g.uncached, Bool(false), - (g.g_type === grantReadExclusive)) - } + def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = + MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> getReleaseType(M_FLUSH, meta), + probeDowngrade -> getReleaseType(M_PRODUCE, meta), + probeCopy -> getReleaseType(M_CLEAN, meta))) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: ManagerMetadata): UInt = { - Mux(a.uncached, getGrantTypeForUncached(a, m), grantReadExclusive) - } - def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.uncached, Grant.getGrantTypeForUncached(a), grantExclusive) - def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedReadBlock -> probeCopy, + Acquire.uncachedWriteBlock -> probeInvalidate, Acquire.uncachedRead -> probeCopy, Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate - )), probeInvalidate) - } - def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate + Acquire.uncachedAtomic -> probeInvalidate)), + probeInvalidate) - def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = + MuxLookup(cmd, probeCopy, Array( + M_FLUSH -> probeInvalidate, + M_PRODUCE -> probeDowngrade)) + + def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) - def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = + + def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) - def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck - def requiresAckForRelease(r: Release) = Bool(false) - def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) - def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) + def requiresProbes(a: Acquire, meta: ManagerMetadata) = + Mux(dir.none(meta.sharers), Bool(false), + Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive + Mux(a.uncached, a.hasData(), Bool(true)))) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) } class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { @@ -404,152 +374,133 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nManagerStates = 2 def nAcquireTypes = 2 def nProbeTypes = 3 - def nReleaseTypes = 7 + def nReleaseTypes = 6 def nGrantTypes = 3 val clientInvalid :: clientShared :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) + //val managerInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) + val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) - val acquireReadShared :: acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) + val acquireShared :: acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: Nil = Enum(UInt(), nGrantTypes) + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantShared :: grantExclusive :: grantExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) - val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive) + val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty) + val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val grantTypesWithData = Vec(grantShared, grantExclusive) - def isHit (cmd: UInt, m: ClientMetadata): Bool = { - Mux(isWriteIntent(cmd), (m.state === clientExclusiveDirty), - (m.state === clientShared || m.state === clientExclusiveDirty)) - } - def isValid (m: ClientMetadata): Bool = { - m.state != clientInvalid - } - def isHit (incoming: Acquire, m: ManagerMetadata) = isValid(m) - def isValid (m: ManagerMetadata) = m.state != masterInvalid + def isValid(meta: ClientMetadata): Bool = meta.state != clientInvalid + def isValid(meta: ManagerMetadata) = meta.state != managerInvalid - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { - (isRead(cmd) && outstanding.uncached) || - (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) - } - def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { - MuxLookup(cmd, (m.state === clientExclusiveDirty), Array( - M_INV -> (m.state === clientExclusiveDirty), - M_CLN -> (m.state === clientExclusiveDirty) - )) - } - def needsWriteback (m: ClientMetadata): Bool = { - needsTransactionOnCacheControl(M_INV, m) - } - def needsWriteback(m: ManagerMetadata) = isValid(m) + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = + (isRead(cmd) && outstanding.uncached) || + (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusive)) - def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = - ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state)) - def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( - MuxLookup(cmd, clientInvalid, Array( - M_INV -> clientInvalid, - M_CLN -> clientShared - ))) - def clientMetadataOnFlush() = clientMetadataOnCacheControl(M_INV) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - Mux(incoming.uncached, clientInvalid, - Mux(incoming.g_type === grantReadShared, clientShared, - clientExclusiveDirty))) - def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( - MuxLookup(incoming.p_type, m.state, Array( - probeInvalidate -> clientInvalid, - probeDowngrade -> clientShared, - probeCopy -> m.state - ))) - def managerMetadataOnFlush = ManagerMetadata(masterInvalid) - def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(masterValid, dir.pop(m.sharers,src)) - MuxBundle(m, Array( - r.is(releaseVoluntaryInvalidateData) -> next, + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) + + def clientMetadataOnFlush = ClientMetadata(clientInvalid) + + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(cmd, meta.state, Array( + M_FLUSH -> clientInvalid, + M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), + clientShared, meta.state)))) + + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = + ClientMetadata( + Mux(incoming.uncached, clientInvalid, + MuxLookup(incoming.g_type, clientInvalid, Array( + grantShared -> clientShared, + grantExclusive -> clientExclusiveDirty, + grantExclusiveAck -> clientExclusiveDirty)))) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(incoming.p_type, meta.state, Array( + probeInvalidate -> clientInvalid, + probeDowngrade -> clientShared, + probeCopy -> meta.state))) + + def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(managerValid, dir.pop(meta.sharers,src)) + MuxBundle(meta, Array( r.is(releaseInvalidateData) -> next, r.is(releaseInvalidateAck) -> next )) } - def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { - val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = ManagerMetadata(masterValid, m.sharers) - Mux(g.uncached, uncached, cached) + + def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = + Mux(g.uncached, + ManagerMetadata(managerValid, meta.sharers), + ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) + + def managerMetadataOnFlush = ManagerMetadata(managerInvalid) + + def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = + ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) + + def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = + Mux(isWriteIntent(cmd), acquireExclusive, acquireShared) + + def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = + Mux(isWriteIntent(cmd), acquireExclusive, outstanding.a_type) + + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) + MuxLookup(cmd, releaseCopyAck, Array( + M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), + M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), + M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) } - def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { - Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) - } - def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = { - Mux(isWriteIntent(cmd), acquireReadExclusive, outstanding.a_type) - } - def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { - val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> releaseInvalidateData, - probeDowngrade -> releaseDowngradeData, - probeCopy -> releaseCopyData - )) - val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( - probeInvalidate -> releaseInvalidateAck, - probeDowngrade -> releaseDowngradeAck, - probeCopy -> releaseCopyAck - )) - Mux(needsWriteback(m), with_data, without_data) - } - - def messageHasData(msg: TileLinkChannel) = msg match { - case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) - case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) - case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) - case _ => Bool(false) - } - def messageUpdatesDataArray(g: Grant): Bool = { - Mux(g.uncached, Bool(false), - (g.g_type === grantReadShared || g.g_type === grantReadExclusive)) - } + def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = + MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> getReleaseType(M_FLUSH, meta), + probeDowngrade -> getReleaseType(M_PRODUCE, meta), + probeCopy -> getReleaseType(M_CLEAN, meta))) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: ManagerMetadata): UInt = { - Mux(a.uncached, getGrantTypeForUncached(a, m), - Mux(a.a_type === acquireReadShared, - Mux(!dir.none(m.sharers), grantReadShared, grantReadExclusive), - grantReadExclusive)) - } - def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.uncached, Grant.getGrantTypeForUncached(a), + Mux(a.a_type === acquireShared, + Mux(!dir.none(meta.sharers), grantShared, grantExclusive), + grantExclusive)) - def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedReadBlock -> probeCopy, + Acquire.uncachedWriteBlock -> probeInvalidate, Acquire.uncachedRead -> probeCopy, Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate - )), - MuxLookup(a.a_type, probeInvalidate, Array( - acquireReadShared -> probeDowngrade, - acquireReadExclusive -> probeInvalidate - ))) - } - def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate + Acquire.uncachedAtomic -> probeInvalidate)), + MuxLookup(a.a_type, probeCopy, Array( + acquireShared -> probeDowngrade, + acquireExclusive -> probeInvalidate))) - def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = + MuxLookup(cmd, probeCopy, Array( + M_FLUSH -> probeInvalidate, + M_PRODUCE -> probeDowngrade)) + + def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) - def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = + + def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) - def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck - def requiresAckForRelease(r: Release) = Bool(false) - def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) && - Mux(dir.one(m.sharers), Bool(true), - Mux(a.uncached, a.a_type != Acquire.uncachedRead, - a.a_type != acquireReadShared)) - def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) + def requiresProbes(a: Acquire, meta: ManagerMetadata) = + Mux(dir.none(meta.sharers), Bool(false), + Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive + Mux(a.uncached, a.hasData(), a.a_type != acquireShared))) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) } class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { @@ -557,157 +508,135 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nManagerStates = 2 def nAcquireTypes = 2 def nProbeTypes = 3 - def nReleaseTypes = 7 - def nGrantTypes = 4 + def nReleaseTypes = 6 + def nGrantTypes = 3 val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) + //val managerInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) + val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) - val acquireReadShared :: acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) + val acquireShared :: acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantShared :: grantExclusive :: grantExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) - val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive) + val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty) + val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val grantTypesWithData = Vec(grantShared, grantExclusive) - def isHit (cmd: UInt, m: ClientMetadata): Bool = { - Mux(isWriteIntent(cmd), (m.state === clientExclusiveClean || m.state === clientExclusiveDirty), - (m.state === clientShared || m.state === clientExclusiveClean || m.state === clientExclusiveDirty)) - } - def isValid (m: ClientMetadata): Bool = { - m.state != clientInvalid - } - def isHit (incoming: Acquire, m: ManagerMetadata) = isValid(m) - def isValid (m: ManagerMetadata) = m.state != masterInvalid + def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid + def isValid (meta: ManagerMetadata) = meta.state != managerInvalid - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = (isRead(cmd) && outstanding.uncached) || - (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) - } - def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { - MuxLookup(cmd, (m.state === clientExclusiveDirty), Array( - M_INV -> (m.state === clientExclusiveDirty), - M_CLN -> (m.state === clientExclusiveDirty) - )) - } - def needsWriteback (m: ClientMetadata): Bool = { - needsTransactionOnCacheControl(M_INV, m) - } - def needsWriteback(m: ManagerMetadata) = isValid(m) + (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusive)) - def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = - ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state)) + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) - def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( - MuxLookup(cmd, clientInvalid, Array( - M_INV -> clientInvalid, - M_CLN -> clientShared - ))) - def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - Mux(incoming.uncached, clientInvalid, - MuxLookup(incoming.g_type, clientInvalid, Array( - grantReadShared -> clientShared, - grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusive, - clientExclusiveDirty, clientExclusiveClean), - grantReadExclusiveAck -> clientExclusiveDirty - )))) - def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( - MuxLookup(incoming.p_type, m.state, Array( - probeInvalidate -> clientInvalid, - probeDowngrade -> clientShared, - probeCopy -> m.state - ))) - def managerMetadataOnFlush = ManagerMetadata(masterInvalid) - def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(masterValid, dir.pop(m.sharers,src)) - MuxBundle(m, Array( - r.is(releaseVoluntaryInvalidateData) -> next, + def clientMetadataOnFlush = ClientMetadata(clientInvalid) + + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(cmd, meta.state, Array( + M_FLUSH -> clientInvalid, + M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), + clientShared, meta.state), + M_CLEAN -> Mux(meta.state === clientExclusiveDirty, clientExclusiveClean, meta.state)))) + + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = + ClientMetadata( + Mux(incoming.uncached, clientInvalid, + MuxLookup(incoming.g_type, clientInvalid, Array( + grantShared -> clientShared, + grantExclusive -> Mux(outstanding.a_type === acquireExclusive, + clientExclusiveDirty, clientExclusiveClean), + grantExclusiveAck -> clientExclusiveDirty)))) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(incoming.p_type, meta.state, Array( + probeInvalidate -> clientInvalid, + probeDowngrade -> clientShared, + probeCopy -> meta.state))) + + def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(managerValid, dir.pop(meta.sharers,src)) + MuxBundle(meta, Array( r.is(releaseInvalidateData) -> next, r.is(releaseInvalidateAck) -> next )) } - def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { - val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = ManagerMetadata(masterValid, m.sharers) - Mux(g.uncached, uncached, cached) + + def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = + Mux(g.uncached, + ManagerMetadata(managerValid, meta.sharers), + ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) + + def managerMetadataOnFlush = ManagerMetadata(managerInvalid) + + def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = + ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) + + def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = + Mux(isWriteIntent(cmd), acquireExclusive, acquireShared) + + def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = + Mux(isWriteIntent(cmd), acquireExclusive, outstanding.a_type) + + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) + MuxLookup(cmd, releaseCopyAck, Array( + M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), + M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), + M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) } - def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { - Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) - } - def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = { - Mux(isWriteIntent(cmd), acquireReadExclusive, outstanding.a_type) - } - def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { - val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> releaseInvalidateData, - probeDowngrade -> releaseDowngradeData, - probeCopy -> releaseCopyData - )) - val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( - probeInvalidate -> releaseInvalidateAck, - probeDowngrade -> releaseDowngradeAck, - probeCopy -> releaseCopyAck - )) - Mux(needsWriteback(m), with_data, without_data) - } - - def messageHasData(msg: TileLinkChannel) = msg match { - case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) - case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) - case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) - case _ => Bool(false) - } - def messageUpdatesDataArray(g: Grant): Bool = { - Mux(g.uncached, Bool(false), - (g.g_type === grantReadShared || g.g_type === grantReadExclusive)) - } + def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = + MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> getReleaseType(M_FLUSH, meta), + probeDowngrade -> getReleaseType(M_PRODUCE, meta), + probeCopy -> getReleaseType(M_CLEAN, meta))) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: ManagerMetadata): UInt = { - Mux(a.uncached, getGrantTypeForUncached(a, m), - Mux(a.a_type === acquireReadShared, - Mux(!dir.none(m.sharers), grantReadShared, grantReadExclusive), - grantReadExclusive)) - } - def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.uncached, Grant.getGrantTypeForUncached(a), + Mux(a.a_type === acquireShared, + Mux(!dir.none(meta.sharers), grantShared, grantExclusive), + grantExclusive)) - def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedReadBlock -> probeCopy, + Acquire.uncachedWriteBlock -> probeInvalidate, Acquire.uncachedRead -> probeCopy, Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate - )), + Acquire.uncachedAtomic -> probeInvalidate)), MuxLookup(a.a_type, probeCopy, Array( - acquireReadShared -> probeDowngrade, - acquireReadExclusive -> probeInvalidate - ))) - } - def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate + acquireShared -> probeDowngrade, + acquireExclusive -> probeInvalidate))) - def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = + MuxLookup(cmd, probeCopy, Array( + M_FLUSH -> probeInvalidate, + M_PRODUCE -> probeDowngrade)) + + def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) - def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = + + def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) - def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck - def requiresAckForRelease(r: Release) = Bool(false) - def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) && - Mux(dir.one(m.sharers), Bool(true), - Mux(a.uncached, a.a_type != Acquire.uncachedRead, - a.a_type != acquireReadShared)) - def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) + def requiresProbes(a: Acquire, meta: ManagerMetadata) = + Mux(dir.none(meta.sharers), Bool(false), + Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive + Mux(a.uncached, a.hasData(), a.a_type != acquireShared))) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) } class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { @@ -715,174 +644,170 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d def nManagerStates = 2 def nAcquireTypes = 3 def nProbeTypes = 4 - def nReleaseTypes = 11 - def nGrantTypes = 5 + def nReleaseTypes = 10 + def nGrantTypes = 4 val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: clientSharedByTwo :: clientMigratoryClean :: clientMigratoryDirty :: Nil = Enum(UInt(), nClientStates) - //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) + //val managerInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) + val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) - val acquireReadShared :: acquireReadExclusive :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) + val acquireShared :: acquireExclusive :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(UInt(), nProbeTypes) - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadExclusiveAck :: grantReadMigratory :: Nil = Enum(UInt(), nGrantTypes) + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(UInt(), nReleaseTypes) + val grantShared :: grantExclusive :: grantExclusiveAck :: grantReadMigratory :: Nil = Enum(UInt(), nGrantTypes) - val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadMigratory) - val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) + val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty, clientSharedByTwo, clientMigratoryClean, clientMigratoryDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty, clientMigratoryClean, clientMigratoryDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty, clientMigratoryDirty) + val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) + val grantTypesWithData = Vec(grantShared, grantExclusive, grantReadMigratory) - def isHit (cmd: UInt, m: ClientMetadata): Bool = { - Mux(isWriteIntent(cmd), Vec(clientExclusiveClean, clientExclusiveDirty, clientMigratoryClean, clientMigratoryDirty).contains(m.state), (m.state != clientInvalid)) - } - def isValid (m: ClientMetadata): Bool = { - m.state != clientInvalid - } - def isHit (incoming: Acquire, m: ManagerMetadata) = isValid(m) - def isValid (m: ManagerMetadata) = m.state != masterInvalid + def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid + def isValid (meta: ManagerMetadata) = meta.state != managerInvalid - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = (isRead(cmd) && outstanding.uncached) || - (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive && outstanding.a_type != acquireInvalidateOthers)) - } - def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { - MuxLookup(cmd, (m.state === clientExclusiveDirty), Array( - M_INV -> Vec(clientExclusiveDirty,clientMigratoryDirty).contains(m.state), - M_CLN -> Vec(clientExclusiveDirty,clientMigratoryDirty).contains(m.state) - )) - } - def needsWriteback (m: ClientMetadata): Bool = { - needsTransactionOnCacheControl(M_INV, m) - } - def needsWriteback(m: ManagerMetadata) = isValid(m) + (isWriteIntent(cmd) && !Vec(acquireExclusive, acquireInvalidateOthers).contains(outstanding.a_type)) - def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata( - Mux(isWrite(cmd), MuxLookup(m.state, clientExclusiveDirty, Array( - clientExclusiveClean -> clientExclusiveDirty, - clientMigratoryClean -> clientMigratoryDirty)), m.state)) - def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( - MuxLookup(cmd, clientInvalid, Array( - M_INV -> clientInvalid, - M_CLN -> clientShared - ))) - def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - Mux(incoming.uncached, clientInvalid, - MuxLookup(incoming.g_type, clientInvalid, Array( - grantReadShared -> clientShared, - grantReadExclusive -> MuxLookup(outstanding.a_type, clientExclusiveDirty, Array( - acquireReadExclusive -> clientExclusiveDirty, - acquireReadShared -> clientExclusiveClean)), - grantReadExclusiveAck -> clientExclusiveDirty, - grantReadMigratory -> MuxLookup(outstanding.a_type, clientMigratoryDirty, Array( - acquireInvalidateOthers -> clientMigratoryDirty, - acquireReadExclusive -> clientMigratoryDirty, - acquireReadShared -> clientMigratoryClean)) - )))) - def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( - MuxLookup(incoming.p_type, m.state, Array( - probeInvalidate -> clientInvalid, - probeInvalidateOthers -> clientInvalid, - probeCopy -> m.state, - probeDowngrade -> MuxLookup(m.state, clientShared, Array( - clientExclusiveClean -> clientSharedByTwo, - clientExclusiveDirty -> clientSharedByTwo, - clientSharedByTwo -> clientShared, - clientMigratoryClean -> clientSharedByTwo, - clientMigratoryDirty -> clientInvalid)) - ))) - def managerMetadataOnFlush = ManagerMetadata(masterInvalid) - def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(masterValid, dir.pop(m.sharers,src)) - MuxBundle(m, Array( - r.is(releaseVoluntaryInvalidateData) -> next, + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + Mux(isWrite(cmd), MuxLookup(meta.state, clientExclusiveDirty, Array( + clientExclusiveClean -> clientExclusiveDirty, + clientMigratoryClean -> clientMigratoryDirty)), + meta.state)) + + def clientMetadataOnFlush = ClientMetadata(clientInvalid) + + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(cmd, meta.state, Array( + M_FLUSH -> clientInvalid, + M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), + clientShared, meta.state), + M_CLEAN -> MuxLookup(meta.state, meta.state, Array( + clientExclusiveDirty -> clientExclusiveClean, + clientMigratoryDirty -> clientMigratoryClean))))) + + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = + ClientMetadata( + Mux(incoming.uncached, clientInvalid, + MuxLookup(incoming.g_type, clientInvalid, Array( + grantShared -> clientShared, + grantExclusive -> MuxLookup(outstanding.a_type, clientExclusiveDirty, Array( + acquireExclusive -> clientExclusiveDirty, + acquireShared -> clientExclusiveClean)), + grantExclusiveAck -> clientExclusiveDirty, + grantReadMigratory -> MuxLookup(outstanding.a_type, clientMigratoryDirty, Array( + acquireInvalidateOthers -> clientMigratoryDirty, + acquireExclusive -> clientMigratoryDirty, + acquireShared -> clientMigratoryClean)))))) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(incoming.p_type, meta.state, Array( + probeInvalidate -> clientInvalid, + probeInvalidateOthers -> clientInvalid, + probeCopy -> meta.state, + probeDowngrade -> MuxLookup(meta.state, clientShared, Array( + clientExclusiveClean -> clientSharedByTwo, + clientExclusiveDirty -> clientSharedByTwo, + clientSharedByTwo -> clientShared, + clientMigratoryClean -> clientSharedByTwo, + clientMigratoryDirty -> clientInvalid))))) + + def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(managerValid, dir.pop(meta.sharers,src)) + MuxBundle(meta, Array( r.is(releaseInvalidateData) -> next, r.is(releaseInvalidateAck) -> next, r.is(releaseInvalidateDataMigratory) -> next, - r.is(releaseInvalidateAckMigratory) -> next - )) - } - def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { - val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = ManagerMetadata(masterValid, m.sharers) - Mux(g.uncached, uncached, cached) + r.is(releaseInvalidateAckMigratory) -> next)) } + def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = + Mux(g.uncached, + ManagerMetadata(managerValid, meta.sharers), + ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) - def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck + def managerMetadataOnFlush = ManagerMetadata(managerInvalid) - def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { - Mux(isWriteIntent(cmd), Mux(m.state === clientInvalid, acquireReadExclusive, acquireInvalidateOthers), acquireReadShared) + def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = + ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) + + def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = + Mux(isWriteIntent(cmd), + Mux(meta.state === clientInvalid, acquireExclusive, acquireInvalidateOthers), + acquireShared) + + def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = + Mux(isWriteIntent(cmd), + Mux(meta.state === clientInvalid, acquireExclusive, acquireInvalidateOthers), + outstanding.a_type) + + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) + MuxLookup(cmd, releaseCopyAck, Array( + M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), + M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), + M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) } - def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = { - Mux(isWriteIntent(cmd), Mux(m.state === clientInvalid, acquireReadExclusive, acquireInvalidateOthers), outstanding.a_type) - } - def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { + + def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> Mux(Vec(clientExclusiveDirty, clientMigratoryDirty).contains(m.state), - releaseInvalidateDataMigratory, releaseInvalidateData), - probeDowngrade -> Mux(m.state === clientMigratoryDirty, releaseDowngradeDataMigratory, releaseDowngradeData), - probeCopy -> releaseCopyData - )) + probeInvalidate -> Mux(Vec(clientExclusiveDirty, clientMigratoryDirty).contains(meta.state), + releaseInvalidateDataMigratory, releaseInvalidateData), + probeDowngrade -> Mux(meta.state === clientMigratoryDirty, + releaseDowngradeDataMigratory, releaseDowngradeData), + probeCopy -> releaseCopyData)) val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( - probeInvalidate -> Mux(clientExclusiveClean === m.state, releaseInvalidateAckMigratory, releaseInvalidateAck), - probeInvalidateOthers -> Mux(m.state === clientSharedByTwo, releaseInvalidateAckMigratory, releaseInvalidateAck), - probeDowngrade -> Mux(m.state != clientInvalid, releaseDowngradeAckHasCopy, releaseDowngradeAck), - probeCopy -> releaseCopyAck - )) - Mux(needsWriteback(m), with_data, without_data) - } - - def messageHasData(msg: TileLinkChannel) = msg match { - case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) - case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) - case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) - case _ => Bool(false) - } - def messageUpdatesDataArray(g: Grant): Bool = { - Mux(g.uncached, Bool(false), - Vec(grantReadShared, grantReadExclusive, grantReadMigratory).contains(g.g_type)) + probeInvalidate -> Mux(clientExclusiveClean === meta.state, + releaseInvalidateAckMigratory, releaseInvalidateAck), + probeInvalidateOthers -> Mux(clientSharedByTwo === meta.state, + releaseInvalidateAckMigratory, releaseInvalidateAck), + probeDowngrade -> Mux(meta.state != clientInvalid, + releaseDowngradeAckHasCopy, releaseDowngradeAck), + probeCopy -> releaseCopyAck)) + Mux(dirty, with_data, without_data) } def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: ManagerMetadata): UInt = { - Mux(a.uncached, getGrantTypeForUncached(a, m), - MuxLookup(a.a_type, grantReadShared, Array( - acquireReadShared -> Mux(!dir.none(m.sharers), grantReadShared, grantReadExclusive), - acquireReadExclusive -> grantReadExclusive, - acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI for broadcast? - ))) - } - def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.uncached, Grant.getGrantTypeForUncached(a), + MuxLookup(a.a_type, grantShared, Array( + acquireShared -> Mux(!dir.none(meta.sharers), grantShared, grantExclusive), + acquireExclusive -> grantExclusive, + acquireInvalidateOthers -> grantExclusiveAck))) //TODO: add this to MESI for broadcast? - def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedReadBlock -> probeCopy, + Acquire.uncachedWriteBlock -> probeInvalidate, Acquire.uncachedRead -> probeCopy, Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate - )), + Acquire.uncachedAtomic -> probeInvalidate)), MuxLookup(a.a_type, probeCopy, Array( - acquireReadShared -> probeDowngrade, - acquireReadExclusive -> probeInvalidate, - acquireInvalidateOthers -> probeInvalidateOthers - ))) - } - def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate + acquireShared -> probeDowngrade, + acquireExclusive -> probeInvalidate, + acquireInvalidateOthers -> probeInvalidateOthers))) - def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = + MuxLookup(cmd, probeCopy, Array( + M_FLUSH -> probeInvalidate, + M_PRODUCE -> probeDowngrade)) + + def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), acq.a_type != acquireInvalidateOthers) - def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = + + def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) - def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck - def requiresAckForRelease(r: Release) = Bool(false) - def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) && - Mux(dir.one(m.sharers), Bool(true), - Mux(a.uncached, a.a_type != Acquire.uncachedRead, - a.a_type != acquireReadShared)) - def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) + def requiresProbes(a: Acquire, meta: ManagerMetadata) = + Mux(dir.none(meta.sharers), Bool(false), + Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive + Mux(a.uncached, a.hasData(), a.a_type != acquireShared))) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) } diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index fe647d07..69cdc41f 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -16,7 +16,6 @@ trait MemoryOpConstants { val MT_BU = Bits("b100") val MT_HU = Bits("b101") val MT_WU = Bits("b110") - val MT_CB = Bits("b111") // cache block val NUM_XA_OPS = 9 val M_SZ = 5 @@ -37,8 +36,9 @@ trait MemoryOpConstants { val M_XA_MAX = Bits("b01101"); val M_XA_MINU = Bits("b01110"); val M_XA_MAXU = Bits("b01111"); - val M_INV = Bits("b10000"); // write back and invalidate line - val M_CLN = Bits("b10001"); // write back line + val M_FLUSH = Bits("b10000") // write back dirty data and cede R/W permissions + val M_PRODUCE = Bits("b10001") // write back dirty data and cede W permissions + val M_CLEAN = Bits("b10011") // write back dirty data and retain R/W permissions def isAMO(cmd: Bits) = cmd(3) || cmd === M_XA_SWAP def isPrefetch(cmd: Bits) = cmd === M_PFR || cmd === M_PFW diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 4db133d6..5e5053be 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -136,7 +136,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { mem_acked := Bool(true) mem_gxid := io.mem.grant.bits.payload.manager_xact_id mem_gsrc := io.mem.grant.bits.header.src - mem_needs_ack := co.requiresAckForGrant(io.mem.grant.bits.payload) + mem_needs_ack := io.mem.grant.bits.payload.requiresAck() } io.mem.grant.ready := Bool(true) @@ -184,22 +184,26 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { state := Mux(cmd === cmd_readmem && pos != UInt(0), state_mem_rreq, state_rx) } - var mem_req_data: UInt = null - for (i <- 0 until dataBits/short_request_bits) { - val idx = Cat(cnt, UInt(i, log2Up(dataBits/short_request_bits))) + val n = dataBits/short_request_bits + val mem_req_data = (0 until n).map { i => + val ui = UInt(i, log2Up(n)) when (state === state_mem_rresp && io.mem.grant.valid) { - packet_ram(idx) := io.mem.grant.bits.payload.data((i+1)*short_request_bits-1, i*short_request_bits) + packet_ram(Cat(io.mem.grant.bits.payload.addr_beat, ui)) := + io.mem.grant.bits.payload.data((i+1)*short_request_bits-1, i*short_request_bits) } - mem_req_data = Cat(packet_ram(idx), mem_req_data) - } + packet_ram(Cat(cnt, ui)) + }.reverse.reduce(_##_) + val init_addr = addr.toUInt >> UInt(offsetBits-3) io.mem.acquire.valid := state === state_mem_rreq || state === state_mem_wreq io.mem.acquire.bits.payload := Mux(cmd === cmd_writemem, - UncachedWrite(init_addr, mem_req_data), - UncachedRead(init_addr)) + UncachedWriteBlock( + addr_block = init_addr, + addr_beat = cnt, + client_xact_id = UInt(0), + data = mem_req_data), + UncachedReadBlock(addr_block = init_addr)) io.mem.acquire.bits.payload.data := mem_req_data - io.mem.acquire.bits.header.src := UInt(params(LNClients)) // By convention HTIF is the client with the largest id - io.mem.acquire.bits.header.dst := UInt(0) // DNC; Overwritten outside module io.mem.finish.valid := (state === state_mem_finish) && mem_needs_ack io.mem.finish.bits.payload.manager_xact_id := mem_gxid io.mem.finish.bits.header.dst := mem_gsrc diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index d6133fc0..e7291222 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -228,7 +228,9 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { mem_cmd_q.io.enq.valid := Bool(false) mem_data_q.io.enq.valid := Bool(false) - val acq_has_data = co.messageHasData(io.uncached.acquire.bits.payload) + //TODO: Assert that only WriteUncachedBlock and ReadUncachedBlock are + //acceptable Acquire types + val acq_has_data = io.uncached.acquire.bits.payload.hasData() val (tl_cnt_out, tl_wrap_out) = Counter(io.uncached.acquire.fire() && acq_has_data, tlDataBeats) val (mif_cnt_out, mif_wrap_out) = Counter(mem_data_q.io.enq.fire(), mifDataBeats) val active_out = Reg(init=Bool(false)) @@ -250,7 +252,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { active_out := Bool(true) cmd_sent_out := Bool(false) tag_out := io.uncached.acquire.bits.payload.client_xact_id - addr_out := io.uncached.acquire.bits.payload.addr + addr_out := io.uncached.acquire.bits.payload.addr_block has_data := acq_has_data tl_done_out := tl_wrap_out mif_done_out := Bool(false) @@ -323,8 +325,12 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { when(tl_wrap_in) { active_in := Bool(false) } } - io.uncached.grant.bits.payload := Grant(Bool(true), Grant.uncachedRead, tag_in, UInt(0), - tl_buf_in(tl_cnt_in)) + io.uncached.grant.bits.payload := Grant(uncached = Bool(true), + g_type = Grant.uncachedReadBlock, + client_xact_id = tag_in, + manager_xact_id = UInt(0), + addr_beat = tl_cnt_in, + data = tl_buf_in(tl_cnt_in)) } class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index af10d455..32ccac87 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -16,20 +16,22 @@ case object TLDataBits extends Field[Int] case object TLDataBeats extends Field[Int] abstract trait TileLinkParameters extends UsesParameters { - val tlAddrBits = params(TLAddrBits) + val tlBlockAddrBits = params(TLAddrBits) val tlClientXactIdBits = params(TLClientXactIdBits) val tlManagerXactIdBits = params(TLManagerXactIdBits) val tlDataBits = params(TLDataBits) val tlDataBeats = params(TLDataBeats) - val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits - val tlSubblockAddrBits = log2Up(tlWriteMaskBits) - val tlAtomicOpcodeBits = log2Up(NUM_XA_OPS) + val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits/8 + val tlBeatAddrBits = log2Up(tlDataBeats) + val tlByteAddrBits = log2Up(tlWriteMaskBits) + val tlAtomicOpcodeBits = M_SZ val tlUncachedOperandSizeBits = MT_SZ val tlSubblockUnionBits = max(tlWriteMaskBits, - (tlSubblockAddrBits + - tlUncachedOperandSizeBits + - tlAtomicOpcodeBits)) + 1 + (tlByteAddrBits + + tlUncachedOperandSizeBits + + tlAtomicOpcodeBits)) + 1 val co = params(TLCoherence) + val networkPreservesPointToPointOrdering = false //TODO: check physical network type } abstract class TLBundle extends Bundle with TileLinkParameters @@ -45,8 +47,12 @@ trait ClientToClientChannel extends TileLinkChannel // Unused for now // Common signals that are used in multiple channels. // These traits are useful for type parameterization. // -trait HasPhysicalAddress extends TLBundle { - val addr = UInt(width = tlAddrBits) +trait HasCacheBlockAddress extends TLBundle { + val addr_block = UInt(width = tlBlockAddrBits) +} + +trait HasTileLinkBeatId extends TLBundle { + val addr_beat = UInt(width = tlBeatAddrBits) } trait HasClientTransactionId extends TLBundle { @@ -57,55 +63,109 @@ trait HasManagerTransactionId extends TLBundle { val manager_xact_id = Bits(width = tlManagerXactIdBits) } -trait HasTileLinkData extends TLBundle { +abstract trait HasTileLinkData extends HasTileLinkBeatId { val data = UInt(width = tlDataBits) + def hasData(dummy: Int = 0): Bool + def hasMultibeatData(dummy: Int = 0): Bool } // Actual TileLink channel bundle definitions class Acquire extends ClientToManagerChannel - with HasPhysicalAddress + with HasCacheBlockAddress with HasClientTransactionId with HasTileLinkData { + // Actual bundle fields val uncached = Bool() - val a_type = UInt(width = max(log2Up(Acquire.nUncachedAcquireTypes), co.acquireTypeWidth)) + val a_type = UInt(width = max(log2Up(Acquire.nBuiltinAcquireTypes), co.acquireTypeWidth)) val subblock = Bits(width = tlSubblockUnionBits) - val sbAddrOff = tlSubblockAddrBits + tlUncachedOperandSizeBits - val opSzOff = tlUncachedOperandSizeBits + sbAddrOff + + // Utility funcs for accessing uncached/subblock union + val opSizeOff = tlByteAddrBits + 1 + val opCodeOff = tlUncachedOperandSizeBits + opSizeOff + val opMSB = tlAtomicOpcodeBits + opCodeOff def allocate(dummy: Int = 0) = subblock(0) - def operand_sz(dummy: Int = 0) = subblock(tlUncachedOperandSizeBits, 1) - def subblock_addr(dummy: Int = 0) = subblock(sbAddrOff, tlUncachedOperandSizeBits+1) - def atomic_op(dummy: Int = 0) = subblock(opSzOff, sbAddrOff+1) + def addr_byte(dummy: Int = 0) = subblock(opSizeOff-1, 1) + def op_size(dummy: Int = 0) = subblock(opCodeOff-1, opSizeOff) + def op_code(dummy: Int = 0) = subblock(opMSB-1, opCodeOff) def write_mask(dummy: Int = 0) = subblock(tlWriteMaskBits, 1) + def addr(dummy: Int = 0) = Cat(addr_block, addr_beat, this.addr_byte(0)) + + // Other helper funcs def is(t: UInt) = a_type === t + + def hasData(dummy: Int = 0): Bool = uncached && Acquire.typesWithData.contains(a_type) + + def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && uncached && + Acquire.typesWithMultibeatData.contains(a_type) + + //TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache: + def requiresSelfProbe(dummy: Int = 0) = uncached && Acquire.requiresSelfProbe(a_type) + + def makeProbe(meta: ManagerMetadata = co.managerMetadataOnFlush): Probe = + Probe(co.getProbeType(this, meta), this.addr_block) + + def makeGrant( + manager_xact_id: UInt, + meta: ManagerMetadata = co.managerMetadataOnFlush, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Grant = { + Grant( + uncached = this.uncached, + g_type = co.getGrantType(this, meta), + client_xact_id = this.client_xact_id, + manager_xact_id = manager_xact_id, + addr_beat = addr_beat, + data = data + ) + } } object Acquire { - val nUncachedAcquireTypes = 3 - //TODO: val uncachedRead :: uncachedWrite :: uncachedAtomic :: Nil = Enum(UInt(), nUncachedAcquireTypes) - def uncachedRead = UInt(0) - def uncachedWrite = UInt(1) - def uncachedAtomic = UInt(2) - def hasData(a_type: UInt) = Vec(uncachedWrite, uncachedAtomic).contains(a_type) - def requiresOuterRead(a_type: UInt) = a_type != uncachedWrite - def requiresOuterWrite(a_type: UInt) = a_type === uncachedWrite + val nBuiltinAcquireTypes = 5 + //TODO: Use Enum + def uncachedRead = UInt(0) + def uncachedReadBlock = UInt(1) + def uncachedWrite = UInt(2) + def uncachedWriteBlock = UInt(3) + def uncachedAtomic = UInt(4) + def typesWithData = Vec(uncachedWrite, uncachedWriteBlock, uncachedAtomic) + def typesWithMultibeatData = Vec(uncachedWriteBlock) + def requiresOuterRead(a_type: UInt) = a_type != uncachedWriteBlock + def requiresOuterWrite(a_type: UInt) = typesWithData.contains(a_type) + //TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache: + def requiresSelfProbe(a_type: UInt) = a_type === uncachedReadBlock - def apply(uncached: Bool, a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt, subblock: UInt): Acquire = { + def fullWriteMask = SInt(-1, width = new Acquire().tlWriteMaskBits).toUInt + + // Most generic constructor + def apply( + uncached: Bool, + a_type: Bits, + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0), + subblock: UInt = UInt(0)): Acquire = { val acq = new Acquire acq.uncached := uncached acq.a_type := a_type - acq.addr := addr acq.client_xact_id := client_xact_id + acq.addr_block := addr_block + acq.addr_beat := addr_beat acq.data := data acq.subblock := subblock acq } - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt): Acquire = { - apply(Bool(false), a_type, addr, client_xact_id, data, UInt(0)) - } - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt): Acquire = { - apply(a_type, addr, client_xact_id, UInt(0)) + // For cached types + def apply(a_type: Bits, client_xact_id: UInt, addr_block: UInt): Acquire = { + apply( + uncached = Bool(false), + a_type = a_type, + client_xact_id = client_xact_id, + addr_block = addr_block) } + // Copy constructor def apply(a: Acquire): Acquire = { val acq = new Acquire acq := a @@ -113,83 +173,189 @@ object Acquire { } } +// Asks for a single TileLink beat of data object UncachedRead { - def apply(addr: UInt, client_xact_id: UInt, subblock_addr: UInt, operand_sz: UInt, alloc: Bool): Acquire = { - val acq = Acquire(Acquire.uncachedRead, addr, client_xact_id) - acq.uncached := Bool(true) - acq.subblock := Cat(subblock_addr, operand_sz, alloc) - acq + def apply( + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, + alloc: Bool = Bool(true)): Acquire = { + Acquire( + uncached = Bool(true), + a_type = Acquire.uncachedRead, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = addr_beat, + subblock = alloc) } - def apply(addr: UInt, client_xact_id: UInt): Acquire = { - apply(addr, client_xact_id, UInt(0), MT_CB, Bool(true)) - } - def apply(addr: UInt): Acquire = { - apply(addr, UInt(0)) +} + +// Asks for an entire cache block of data +object UncachedReadBlock { + def apply( + client_xact_id: UInt = UInt(0), + addr_block: UInt, + alloc: Bool = Bool(true)): Acquire = { + Acquire( + uncached = Bool(true), + a_type = Acquire.uncachedReadBlock, + client_xact_id = client_xact_id, + addr_block = addr_block, + subblock = alloc.toUInt) } } object UncachedWrite { - def apply(addr: UInt, client_xact_id: UInt, write_mask: Bits, alloc: Bool, data: UInt): Acquire = { - val acq = Acquire(Acquire.uncachedWrite, addr, client_xact_id, data) - acq.uncached := Bool(true) - acq.subblock := Cat(write_mask, alloc) - acq + def apply( + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, + data: UInt, + write_mask: UInt = Acquire.fullWriteMask, + alloc: Bool = Bool(true)): Acquire = { + Acquire( + uncached = Bool(true), + a_type = Acquire.uncachedWrite, + addr_block = addr_block, + addr_beat = addr_beat, + client_xact_id = client_xact_id, + data = data, + subblock = Cat(write_mask, alloc)) } - def apply(addr: UInt, client_xact_id: UInt, data: UInt): Acquire = { - apply(addr, client_xact_id, SInt(-1), Bool(true), data) - } - def apply(addr: UInt, data: UInt): Acquire = { - apply(addr, UInt(0), data) +} + +// For full block of data +object UncachedWriteBlock { + def apply( + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, + data: UInt, + alloc: Bool = Bool(true)): Acquire = { + Acquire( + uncached = Bool(true), + a_type = Acquire.uncachedWriteBlock, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = addr_beat, + data = data, + subblock = Cat(Acquire.fullWriteMask, alloc)) } } object UncachedAtomic { - def apply(addr: UInt, client_xact_id: UInt, atomic_opcode: UInt, - subblock_addr: UInt, operand_sz: UInt, data: UInt): Acquire = { - val acq = Acquire(Acquire.uncachedAtomic, addr, client_xact_id, data) - acq.uncached := Bool(true) - acq.subblock := Cat(atomic_opcode, subblock_addr, operand_sz, Bool(true)) - acq + def apply( + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, + addr_byte: UInt, + atomic_opcode: UInt, + operand_size: UInt, + data: UInt): Acquire = { + Acquire( + uncached = Bool(true), + a_type = Acquire.uncachedAtomic, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = addr_beat, + data = data, + subblock = Cat(atomic_opcode, operand_size, addr_byte, Bool(true))) } } class Probe extends ManagerToClientChannel - with HasPhysicalAddress { + with HasCacheBlockAddress { val p_type = UInt(width = co.probeTypeWidth) + def is(t: UInt) = p_type === t + def makeRelease( + client_xact_id: UInt, + meta: ClientMetadata = co.clientMetadataOnFlush, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Release = { + Release( + voluntary = Bool(false), + r_type = co.getReleaseType(this, meta), + client_xact_id = client_xact_id, + addr_block = this.addr_block, + addr_beat = addr_beat, + data = data) + } } object Probe { - def apply(p_type: UInt, addr: UInt) = { + val co = new Probe().co + def apply(p_type: UInt, addr_block: UInt) = { val prb = new Probe prb.p_type := p_type - prb.addr := addr + prb.addr_block := addr_block prb } + + def onVoluntaryWriteback(meta: ManagerMetadata, addr_block: UInt): Probe = { + apply(co.getProbeType(M_FLUSH, meta), addr_block) + } } + class Release extends ClientToManagerChannel - with HasPhysicalAddress + with HasCacheBlockAddress with HasClientTransactionId with HasTileLinkData { val r_type = UInt(width = co.releaseTypeWidth) + val voluntary = Bool() + + // Helper funcs def is(t: UInt) = r_type === t + def hasData(dummy: Int = 0) = co.releaseTypesWithData.contains(r_type) + def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) && co.releaseTypesWithData.contains(r_type) + def isVoluntary(dummy: Int = 0) = voluntary + def requiresAck(dummy: Int = 0) = !Bool(networkPreservesPointToPointOrdering) + + def makeGrant( + manager_xact_id: UInt, + meta: ManagerMetadata = co.managerMetadataOnFlush): Grant = { + Grant( + g_type = Grant.voluntaryAck, + uncached = Bool(true), // Grant.voluntaryAck is built-in type + client_xact_id = this.client_xact_id, + manager_xact_id = manager_xact_id + ) + } } object Release { - def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, data: UInt): Release = { + val co = new Release().co + def apply( + voluntary: Bool, + r_type: UInt, + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Release = { val rel = new Release rel.r_type := r_type - rel.addr := addr rel.client_xact_id := client_xact_id + rel.addr_block := addr_block + rel.addr_beat := addr_beat rel.data := data + rel.voluntary := voluntary rel } - def apply(r_type: UInt, addr: UInt, client_xact_id: UInt): Release = { - apply(r_type, addr, client_xact_id, UInt(0)) - } - def apply(r_type: UInt, addr: UInt): Release = { - apply(r_type, addr, UInt(0), UInt(0)) + + def makeVoluntaryWriteback( + meta: ClientMetadata, + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Release = { + Release( + voluntary = Bool(true), + r_type = co.getReleaseType(M_FLUSH, meta), + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = addr_beat, + data = data) } } @@ -198,29 +364,62 @@ class Grant extends ManagerToClientChannel with HasClientTransactionId with HasManagerTransactionId { val uncached = Bool() - val g_type = UInt(width = max(log2Up(Grant.nUncachedGrantTypes), co.grantTypeWidth)) + val g_type = UInt(width = max(log2Up(Grant.nBuiltinGrantTypes), co.grantTypeWidth)) + + // Helper funcs def is(t: UInt) = g_type === t + def hasData(dummy: Int = 0): Bool = Mux(uncached, + Grant.typesWithData.contains(g_type), + co.grantTypesWithData.contains(g_type)) + def hasMultibeatData(dummy: Int = 0): Bool = + Bool(tlDataBeats > 1) && Mux(uncached, + Grant.typesWithMultibeatData.contains(g_type), + co.grantTypesWithData.contains(g_type)) + def isVoluntary(dummy: Int = 0): Bool = uncached && (g_type === Grant.voluntaryAck) + def requiresAck(dummy: Int = 0): Bool = !Bool(networkPreservesPointToPointOrdering) && !isVoluntary() + def makeFinish(dummy: Int = 0): Finish = { + val f = new Finish + f.manager_xact_id := this.manager_xact_id + f + } } object Grant { - val nUncachedGrantTypes = 3 - //TODO val uncachedRead :: uncachedWrite :: uncachedAtomic :: Nil = Enum(UInt(), nUncachedGrantTypes) - def uncachedRead = UInt(0) - def uncachedWrite = UInt(1) - def uncachedAtomic = UInt(2) - def hasData(g_type: UInt) = Vec(uncachedRead, uncachedAtomic).contains(g_type) + val nBuiltinGrantTypes = 5 + //TODO Use Enum + def voluntaryAck = UInt(0) + def uncachedRead = UInt(1) + def uncachedReadBlock = UInt(2) + def uncachedWrite = UInt(3) + def uncachedAtomic = UInt(4) + def typesWithData = Vec(uncachedRead, uncachedReadBlock, uncachedAtomic) + def typesWithMultibeatData= Vec(uncachedReadBlock) - def apply(uncached: Bool, g_type: UInt, client_xact_id: UInt, manager_xact_id: UInt, data: UInt): Grant = { + def apply( + uncached: Bool, + g_type: UInt, + client_xact_id: UInt, + manager_xact_id: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Grant = { val gnt = new Grant gnt.uncached := uncached gnt.g_type := g_type gnt.client_xact_id := client_xact_id gnt.manager_xact_id := manager_xact_id + gnt.addr_beat := addr_beat gnt.data := data gnt } - def apply(uncached: Bool, g_type: UInt, client_xact_id: UInt, manager_xact_id: UInt): Grant = { - apply(uncached, g_type, client_xact_id, manager_xact_id, UInt(0)) + + def getGrantTypeForUncached(a: Acquire): UInt = { + MuxLookup(a.a_type, Grant.uncachedRead, Array( + Acquire.uncachedRead -> Grant.uncachedRead, + Acquire.uncachedReadBlock -> Grant.uncachedReadBlock, + Acquire.uncachedWrite -> Grant.uncachedWrite, + Acquire.uncachedWriteBlock -> Grant.uncachedWrite, + Acquire.uncachedAtomic -> Grant.uncachedAtomic + )) } } @@ -260,10 +459,17 @@ object TileLinkIOWrapper { } } -// Utility functions for constructing TileLinkIO arbiters -abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule { +abstract trait HasArbiterTypes { + val arbN: Int type ManagerSourcedWithId = ManagerToClientChannel with HasClientTransactionId - type ClientSourcedWithId = ClientToManagerChannel with HasClientTransactionId + type ClientSourcedWithId = ClientToManagerChannel with HasClientTransactionId + type ClientSourcedWithIdAndData = ClientToManagerChannel with + HasClientTransactionId with + HasTileLinkData +} +// Utility functions for constructing TileLinkIO arbiters +abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule + with HasArbiterTypes { // These are filled in depending on whether the arbiter mucks with the // client ids and then needs to revert them on the way back @@ -271,10 +477,10 @@ abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule { def managerSourcedClientXactId(in: ManagerSourcedWithId): Bits def arbIdx(in: ManagerSourcedWithId): UInt - def hookupClientSource[M <: ClientSourcedWithId] + def hookupClientSource[M <: ClientSourcedWithIdAndData] (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], out: DecoupledIO[LogicalNetworkIO[M]]) { - def hasData(m: LogicalNetworkIO[M]) = co.messageHasData(m.payload) + def hasData(m: LogicalNetworkIO[M]) = m.payload.hasMultibeatData() val arb = Module(new LockingRRArbiter(out.bits.clone, arbN, params(TLDataBeats), Some(hasData _))) out <> arb.io.out ins.zipWithIndex.zip(arb.io.in).map{ case ((req,id), arb) => { @@ -336,35 +542,26 @@ abstract class TileLinkIOArbiter(n: Int) extends TileLinkArbiterLike(n) { } // Appends the port index of the arbiter to the client_xact_id -abstract trait AppendsArbiterId { - val arbN: Int - def clientSourcedClientXactId(in: ClientToManagerChannel with HasClientTransactionId, id: Int) = +abstract trait AppendsArbiterId extends HasArbiterTypes { + def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = Cat(in.client_xact_id, UInt(id, log2Up(arbN))) - def managerSourcedClientXactId(in: ManagerToClientChannel with HasClientTransactionId) = + def managerSourcedClientXactId(in: ManagerSourcedWithId) = in.client_xact_id >> UInt(log2Up(arbN)) - def arbIdx(in: ManagerToClientChannel with HasClientTransactionId) = - in.client_xact_id(log2Up(arbN)-1,0).toUInt + def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id(log2Up(arbN)-1,0).toUInt } // Uses the client_xact_id as is (assumes it has been set to port index) -abstract trait PassesId { - def clientSourcedClientXactId(in: ClientToManagerChannel with HasClientTransactionId, id: Int) = - in.client_xact_id - def managerSourcedClientXactId(in: ManagerToClientChannel with HasClientTransactionId) = - in.client_xact_id - def arbIdx(in: ManagerToClientChannel with HasClientTransactionId) = - in.client_xact_id +abstract trait PassesId extends HasArbiterTypes { + def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = in.client_xact_id + def managerSourcedClientXactId(in: ManagerSourcedWithId) = in.client_xact_id + def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id } // Overwrites some default client_xact_id with the port idx -abstract trait UsesNewId { - val arbN: Int - def clientSourcedClientXactId(in: ClientToManagerChannel with HasClientTransactionId, id: Int) = - UInt(id, log2Up(arbN)) - def managerSourcedClientXactId(in: ManagerToClientChannel with HasClientTransactionId) = - UInt(0) - def arbIdx(in: ManagerToClientChannel with HasClientTransactionId) = - in.client_xact_id +abstract trait UsesNewId extends HasArbiterTypes { + def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = UInt(id, log2Up(arbN)) + def managerSourcedClientXactId(in: ManagerSourcedWithId) = UInt(0) + def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id } // Mix-in id generation traits to make concrete arbiter classes diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index d2531a9e..c05aafee 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -67,7 +67,7 @@ class L2BroadcastHub(bankId: Int, innerId: String, outerId: String) extends val sdq_val = Reg(init=Bits(0, sdqDepth)) val sdq_alloc_id = PriorityEncoder(~sdq_val) val sdq_rdy = !sdq_val.andR - val sdq_enq = acquire.fire() && co.messageHasData(acquire.bits.payload) + val sdq_enq = acquire.fire() && acquire.bits.payload.hasData() val sdq = Vec.fill(sdqDepth){ Reg(io.inner.acquire.bits.payload.data) } when (sdq_enq) { sdq(sdq_alloc_id) := acquire.bits.payload.data } @@ -87,8 +87,8 @@ class L2BroadcastHub(bankId: Int, innerId: String, outerId: String) extends // Queue to store impending Voluntary Release data val release = io.inner.release - val voluntary = co.isVoluntary(release.bits.payload) - val vwbdq_enq = release.fire() && voluntary && co.messageHasData(release.bits.payload) + val voluntary = release.bits.payload.isVoluntary() + val vwbdq_enq = release.fire() && voluntary && release.bits.payload.hasData() val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, tlDataBeats) //TODO Zero width val vwbdq = Vec.fill(tlDataBeats){ Reg(release.bits.payload.data) } //TODO Assumes nReleaseTransactors == 1 when(vwbdq_enq) { vwbdq(rel_data_cnt) := release.bits.payload.data } @@ -117,9 +117,10 @@ class L2BroadcastHub(bankId: Int, innerId: String, outerId: String) extends probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.probe } // Wire grant reply to initiating client - def hasData(m: LogicalNetworkIO[Grant]) = co.messageHasData(m.payload) + def hasData(m: LogicalNetworkIO[Grant]) = m.payload.hasMultibeatData() val grant_arb = Module(new LockingArbiter(new LogicalNetworkIO(new Grant), trackerList.size, tlDataBeats, Some(hasData _))) io.inner.grant.bits.payload.data := io.outer.grant.bits.payload.data + io.inner.grant.bits.payload.addr_beat := io.outer.grant.bits.payload.addr_beat io.inner.grant <> grant_arb.io.out grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } @@ -130,7 +131,7 @@ class L2BroadcastHub(bankId: Int, innerId: String, outerId: String) extends val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.payload.data) val is_in_sdq = outer_data_ptr.loc === inStoreQueue val free_sdq = io.outer.acquire.fire() && - co.messageHasData(io.outer.acquire.bits.payload) && + io.outer.acquire.bits.payload.hasData() && outer_data_ptr.loc === inStoreQueue io.outer.acquire.bits.payload.data := MuxLookup(outer_data_ptr.loc, release.bits.payload.data, Array( inStoreQueue -> sdq(outer_data_ptr.idx), @@ -156,32 +157,39 @@ abstract class XactTracker(innerId: String, outerId: String) extends Module { val has_release_match = Bool(OUTPUT) } - val c_acq = io.inner.acquire.bits - val c_rel = io.inner.release.bits - val c_gnt = io.inner.grant.bits - val c_ack = io.inner.finish.bits - val m_gnt = io.outer.grant.bits + val cacq = io.inner.acquire.bits + val crel = io.inner.release.bits + val cgnt = io.inner.grant.bits + val cfin = io.inner.finish.bits + val macq = io.outer.acquire.bits + val mgnt = io.outer.grant.bits } class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { - val s_idle :: s_outer :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) + val s_idle :: s_outer :: s_grant :: s_ack :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) val xact_src = Reg(io.inner.release.bits.header.src.clone) val xact_r_type = Reg(io.inner.release.bits.payload.r_type) - val xact_addr = Reg(io.inner.release.bits.payload.addr.clone) + val xact_addr_block = Reg(io.inner.release.bits.payload.addr_block.clone) val xact_client_xact_id = Reg(io.inner.release.bits.payload.client_xact_id.clone) val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) } + val xact = Release( + voluntary = Bool(true), + r_type = xact_r_type, + client_xact_id = xact_client_xact_id, + addr_block = xact_addr_block) val collect_inner_data = Reg(init=Bool(false)) + // TODO: assert that all releases have full blocks of data val (inner_data_cnt, inner_data_done) = - Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) + Counter(io.inner.release.fire() && io.inner.release.bits.payload.hasMultibeatData(), tlDataBeats) val (outer_data_cnt, outer_data_done) = - Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) + Counter(io.outer.acquire.fire() && io.outer.acquire.bits.payload.hasMultibeatData(), tlDataBeats) io.has_acquire_conflict := Bool(false) - io.has_release_match := co.isVoluntary(c_rel.payload) + io.has_release_match := crel.payload.isVoluntary() io.outer.grant.ready := Bool(false) io.outer.acquire.valid := Bool(false) @@ -193,21 +201,19 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := Grant(Bool(false), - co.getGrantTypeOnVoluntaryWriteback(co.managerMetadataOnFlush), - xact_client_xact_id, - UInt(trackerId)) + io.inner.grant.bits.payload := xact.makeGrant(UInt(trackerId)) - io.outer.acquire.bits.payload := Bundle(UncachedWrite( - xact_addr, - UInt(trackerId), - xact_data(outer_data_cnt)), + io.outer.acquire.bits.payload := Bundle(UncachedWriteBlock( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block, + addr_beat = outer_data_cnt, + data = xact_data(outer_data_cnt)), { case TLId => outerId }) when(collect_inner_data) { io.inner.release.ready := Bool(true) when(io.inner.release.valid) { - xact_data(inner_data_cnt) := c_rel.payload.data + xact_data(inner_data_cnt) := crel.payload.data } when(inner_data_done) { collect_inner_data := Bool(false) } } @@ -216,72 +222,107 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute is(s_idle) { io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { - xact_src := c_rel.header.src - xact_r_type := c_rel.payload.r_type - xact_addr := c_rel.payload.addr - xact_client_xact_id := c_rel.payload.client_xact_id - xact_data(UInt(0)) := c_rel.payload.data - collect_inner_data := co.messageHasData(c_rel.payload) - state := Mux(co.messageHasData(c_rel.payload), s_outer, s_ack) + xact_src := crel.header.src + xact_r_type := crel.payload.r_type + xact_addr_block := crel.payload.addr_block + xact_client_xact_id := crel.payload.client_xact_id + xact_data(UInt(0)) := crel.payload.data + collect_inner_data := crel.payload.hasMultibeatData() + state := Mux(crel.payload.hasData(), s_outer, + Mux(crel.payload.requiresAck(), s_ack, s_idle)) } } is(s_outer) { io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) - when(outer_data_done) { state := s_ack } + when(outer_data_done) { + state := Mux(xact.requiresAck(), s_grant, s_idle) + } + } + is(s_grant) { + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { + state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) + } } is(s_ack) { - io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { state := s_idle } + // TODO: This state is unnecessary if no client will ever issue the + // pending Acquire that caused this writeback until it receives the + // Grant.voluntaryAck for this writeback + io.inner.finish.ready := Bool(true) + when(io.inner.finish.valid) { state := s_idle } } } } class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { - val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) + val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_mem_resp :: s_ack :: Nil = Enum(UInt(), 7) val state = Reg(init=s_idle) val xact_src = Reg(io.inner.acquire.bits.header.src.clone) val xact_uncached = Reg(io.inner.acquire.bits.payload.uncached.clone) val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone) - val xact_addr = Reg(io.inner.acquire.bits.payload.addr.clone) val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone) + val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone) + val xact_addr_beat = Reg(io.inner.acquire.bits.payload.addr_beat.clone) val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone) val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } - val xact = Acquire(xact_uncached, xact_a_type, xact_addr, xact_client_xact_id, UInt(0), xact_subblock) + val xact = Acquire( + uncached = xact_uncached, + a_type = xact_a_type, + client_xact_id = xact_client_xact_id, + addr_block = xact_addr_block, + addr_beat = xact_addr_beat, + data = UInt(0), + subblock = xact_subblock) val collect_inner_data = Reg(init=Bool(false)) + //TODO: Assert that if xact.uncached, xact_a_type is ReadBlock or WriteBlock val (inner_data_cnt, inner_data_done) = - Counter(io.inner.acquire.fire() && co.messageHasData(io.inner.acquire.bits.payload), tlDataBeats) + Counter(io.inner.acquire.fire() && cacq.payload.hasMultibeatData(), tlDataBeats) val (outer_data_cnt, outer_data_done) = - Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) + Counter(io.outer.acquire.fire() && macq.payload.hasMultibeatData(), tlDataBeats) + val (cgnt_data_cnt, cgnt_data_done) = + Counter(io.inner.grant.fire() && cgnt.payload.hasMultibeatData(), tlDataBeats) + val release_count = Reg(init=UInt(0, width = log2Up(nClients))) val probe_flags = Reg(init=Bits(0, width = nClients)) val curr_p_id = PriorityEncoder(probe_flags) - val pending_outer_write = co.messageHasData(xact) + val pending_outer_write = xact.hasData() val pending_outer_read = co.requiresOuterRead(xact, co.managerMetadataOnFlush) val probe_initial_flags = Bits(width = nClients) probe_initial_flags := Bits(0) // issue self-probes for uncached read xacts to facilitate I$ coherence - val probe_self = co.requiresSelfProbe(io.inner.acquire.bits.payload) - val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(nClients)-1,0))) + val probe_self = io.inner.acquire.bits.payload.requiresSelfProbe() + val myflag = Mux(probe_self, Bits(0), UIntToOH(cacq.header.src(log2Up(nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) - io.has_acquire_conflict := co.isCoherenceConflict(xact_addr, c_acq.payload.addr) && + io.has_acquire_conflict := co.isCoherenceConflict(xact_addr_block, cacq.payload.addr_block) && (state != s_idle) && !collect_inner_data - io.has_release_match := co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && - !co.isVoluntary(c_rel.payload) && + io.has_release_match := co.isCoherenceConflict(xact_addr_block, crel.payload.addr_block) && + !crel.payload.isVoluntary() && (state != s_idle) - val outer_write_acq = Bundle(UncachedWrite(xact_addr, UInt(trackerId), xact_data(outer_data_cnt)), + val outer_write_acq = Bundle(UncachedWriteBlock( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block, + addr_beat = outer_data_cnt, + data = xact_data(outer_data_cnt)), { case TLId => outerId }) - val outer_write_rel = Bundle(UncachedWrite(xact_addr, UInt(trackerId), c_rel.payload.data), + val outer_write_rel = Bundle(UncachedWriteBlock( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block, + addr_beat = crel.payload.addr_beat, + data = crel.payload.data), + { case TLId => outerId }) + val outer_read = Bundle(UncachedReadBlock( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block), { case TLId => outerId }) - val outer_read = Bundle(UncachedRead(xact_addr, UInt(trackerId)), { case TLId => outerId }) io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.payload := outer_read //default @@ -290,16 +331,12 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.managerMetadataOnFlush), xact_addr) + io.inner.probe.bits.payload := xact.makeProbe() io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := Grant(xact_uncached, - co.getGrantType(xact, co.managerMetadataOnFlush), - xact_client_xact_id, - UInt(trackerId), - UInt(0)) // Data bypassed in parent + io.inner.grant.bits.payload := xact.makeGrant(UInt(trackerId)) // Data bypassed in parent io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) @@ -307,7 +344,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri when(collect_inner_data) { io.inner.acquire.ready := Bool(true) when(io.inner.acquire.valid) { - xact_data(inner_data_cnt) := c_acq.payload.data + xact_data(inner_data_cnt) := cacq.payload.data } when(inner_data_done) { collect_inner_data := Bool(false) } } @@ -315,17 +352,18 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri switch (state) { is(s_idle) { io.inner.acquire.ready := Bool(true) - val needs_outer_write = co.messageHasData(c_acq.payload) - val needs_outer_read = co.requiresOuterRead(c_acq.payload, co.managerMetadataOnFlush) - when( io.inner.acquire.valid ) { - xact_uncached := c_acq.payload.uncached - xact_a_type := c_acq.payload.a_type - xact_addr := c_acq.payload.addr - xact_client_xact_id := c_acq.payload.client_xact_id - xact_data(UInt(0)) := c_acq.payload.data - xact_subblock := c_acq.payload.subblock - xact_src := c_acq.header.src - collect_inner_data := co.messageHasData(c_acq.payload) + val needs_outer_write = cacq.payload.hasData() + val needs_outer_read = co.requiresOuterRead(cacq.payload, co.managerMetadataOnFlush) + when(io.inner.acquire.valid) { + xact_uncached := cacq.payload.uncached + xact_a_type := cacq.payload.a_type + xact_addr_block := cacq.payload.addr_block + xact_addr_beat := cacq.payload.addr_beat + xact_client_xact_id := cacq.payload.client_xact_id + xact_data(UInt(0)) := cacq.payload.data + xact_subblock := cacq.payload.subblock + xact_src := cacq.header.src + collect_inner_data := cacq.payload.hasMultibeatData() probe_flags := probe_initial_flags release_count := PopCount(probe_initial_flags) state := Mux(probe_initial_flags.orR, s_probe, @@ -341,9 +379,9 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri } // Handle releases, which may have data to be written back - io.inner.release.ready := !co.messageHasData(c_rel.payload) || io.outer.acquire.ready + io.inner.release.ready := !crel.payload.hasData() || io.outer.acquire.ready when(io.inner.release.valid) { - when(co.messageHasData(c_rel.payload)) { + when(crel.payload.hasData()) { io.outer.acquire.valid := Bool(true) io.outer.acquire.bits.payload := outer_write_rel when(io.outer.acquire.ready) { @@ -364,14 +402,12 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri } } } - is(s_mem_read) { + is(s_mem_read) { // Read data from outer memory (possibly what was just written) io.outer.acquire.valid := Bool(true) io.outer.acquire.bits.payload := outer_read - when(io.outer.acquire.ready) { - state := Mux(co.requiresAckForGrant(io.inner.grant.bits.payload), s_busy, s_idle) - } + when(io.outer.acquire.ready) { state := s_mem_resp } } - is(s_mem_write) { + is(s_mem_write) { // Write data to outer memory io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) io.outer.acquire.bits.payload := outer_write_acq @@ -379,17 +415,22 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri state := Mux(pending_outer_read, s_mem_read, s_make_grant) } } - is(s_make_grant) { + is(s_make_grant) { // Manufactor a local grant (some kind of permission upgrade) io.inner.grant.valid := Bool(true) when(io.inner.grant.ready) { - state := Mux(co.requiresAckForGrant(io.inner.grant.bits.payload), s_busy, s_idle) + state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) } } - is(s_busy) { // Nothing left to do but wait for transaction to complete - when(io.outer.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { + is(s_mem_resp) { // Wait to forward grants from outer memory + when(io.outer.grant.valid && mgnt.payload.client_xact_id === UInt(trackerId)) { io.inner.grant.valid := Bool(true) } - when(io.inner.finish.valid && c_ack.payload.manager_xact_id === UInt(trackerId)) { + when(cgnt_data_done) { + state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) + } + } + is(s_ack) { // Wait for transaction to complete + when(io.inner.finish.valid && cfin.payload.manager_xact_id === UInt(trackerId)) { state := s_idle } } diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util.scala index c28778aa..14575aa5 100644 --- a/uncore/src/main/scala/util.scala +++ b/uncore/src/main/scala/util.scala @@ -34,7 +34,7 @@ object ZCounter { } } -class FlowThroughSerializer[T <: HasTileLinkData](gen: LogicalNetworkIO[T], n: Int, doSer: T => Bool) extends Module { +class FlowThroughSerializer[T <: HasTileLinkData](gen: LogicalNetworkIO[T], n: Int) extends Module { val io = new Bundle { val in = Decoupled(gen.clone).flip val out = Decoupled(gen.clone) @@ -65,12 +65,12 @@ class FlowThroughSerializer[T <: HasTileLinkData](gen: LogicalNetworkIO[T], n: I io.out.valid := active || io.in.valid io.out.bits := io.in.bits when(!active && io.in.valid) { - when(doSer(io.in.bits.payload)) { + when(io.in.bits.payload.hasData()) { cnt := Mux(io.out.ready, UInt(1), UInt(0)) rbits := io.in.bits active := Bool(true) } - io.done := !doSer(io.in.bits.payload) + io.done := !io.in.bits.payload.hasData() } when(active) { io.out.bits := rbits From e6491d351f8d9141dbfcf39b30b569ede8e4a050 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 2 Feb 2015 00:22:21 -0800 Subject: [PATCH 281/374] Offset AMOs within beat and return old value --- uncore/src/main/scala/cache.scala | 33 +++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 6651056f..609c3b33 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -166,12 +166,15 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgentParameters { val idxMSB = idxBits-1 val idxLSB = 0 - val refillCyclesPerBeat = params(TLDataBits)/rowBits - val refillCycles = refillCyclesPerBeat*params(TLDataBeats) + val refillCyclesPerBeat = tlDataBits/rowBits + val refillCycles = refillCyclesPerBeat*tlDataBeats require(refillCyclesPerBeat == 1) + val amoAluOperandBits = params(AmoAluOperandBits) + require(amoAluOperandBits <= tlDataBits) } abstract class L2HellaCacheBundle extends TLBundle with L2HellaCacheParameters + abstract class L2HellaCacheModule extends TLModule with L2HellaCacheParameters { def connectDataBeatCounter[S <: HasTileLinkData](inc: Bool, data: S) = { val (cnt, cnt_done) = @@ -750,7 +753,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val xact_addr_beat = Reg(io.inner.acquire.bits.payload.addr_beat.clone) val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone) val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone) - val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } + val xact_data = Vec.fill(tlDataBeats+1) { // Extra entry holds AMO result + Reg(io.inner.acquire.bits.payload.data.clone) + } val xact_tag_match = Reg{ Bool() } val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } @@ -803,13 +808,19 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St def mergeData[T <: HasTileLinkData](buffer: Vec[UInt], incoming: T) { val old_data = incoming.data val new_data = buffer(incoming.addr_beat) - amoalu.io.lhs := old_data - amoalu.io.rhs := new_data - val wmask = FillInterleaved(8, xact.write_mask()) - buffer(incoming.addr_beat) := - Mux(xact.is(Acquire.uncachedAtomic), amoalu.io.out, + val amoOpSz = UInt(amoAluOperandBits) + val offset = xact.addr_byte()(tlByteAddrBits-1, log2Up(amoAluOperandBits/8)) + amoalu.io.lhs := old_data >> offset*amoOpSz + amoalu.io.rhs := new_data >> offset*amoOpSz + val wmask = + Mux(xact.is(Acquire.uncachedAtomic), + FillInterleaved(amoAluOperandBits, UIntToOH(offset)), Mux(xact.is(Acquire.uncachedWriteBlock) || xact.is(Acquire.uncachedWrite), - wmask & new_data | ~wmask & old_data, old_data)) + FillInterleaved(8, xact.write_mask()), + UInt(0, width = tlDataBits))) + buffer(incoming.addr_beat) := ~wmask & old_data | wmask & + Mux(xact.is(Acquire.uncachedAtomic), amoalu.io.out << offset*amoOpSz, new_data) + when(xact.is(Acquire.uncachedAtomic)) { buffer(tlDataBeats) := old_data } // For AMO result } //TODO: Allow hit under miss for stores @@ -860,7 +871,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St manager_xact_id = UInt(trackerId), meta = xact_meta.coh, addr_beat = cgnt_data_cnt, - data = xact_data(cgnt_data_cnt)) + data = Mux(xact.is(Acquire.uncachedAtomic), + xact_data(tlDataBeats), + xact_data(cgnt_data_cnt))) io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) From 6141b3efc53a2c27a50ce28ce3200a79bcc1eb63 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 2 Feb 2015 01:02:06 -0800 Subject: [PATCH 282/374] uncached -> builtin_type --- uncore/src/main/scala/cache.scala | 14 ++--- uncore/src/main/scala/coherence.scala | 80 +++++++++++++-------------- uncore/src/main/scala/memserdes.scala | 2 +- uncore/src/main/scala/tilelink.scala | 47 ++++++++-------- uncore/src/main/scala/uncore.scala | 6 +- 5 files changed, 75 insertions(+), 74 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 609c3b33..794e3447 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -747,7 +747,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val state = Reg(init=s_idle) val xact_src = Reg(io.inner.acquire.bits.header.src.clone) - val xact_uncached = Reg(io.inner.acquire.bits.payload.uncached.clone) + val xact_builtin_type = Reg(io.inner.acquire.bits.payload.builtin_type.clone) val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone) val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone) val xact_addr_beat = Reg(io.inner.acquire.bits.payload.addr_beat.clone) @@ -760,7 +760,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } val xact = Acquire( - uncached = xact_uncached, + builtin_type = xact_builtin_type, a_type = xact_a_type, client_xact_id = xact_client_xact_id, addr_block = xact_addr_block, @@ -796,7 +796,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val needs_writeback = !xact_tag_match && co.isValid(xact_meta.coh) // TODO: dirty bit val is_hit = xact_tag_match && co.isHit(xact, xact_meta.coh) val needs_probes = co.requiresProbes(xact, xact_meta.coh) - //val do_allocate = !xact_uncached || xact.allocate() + //val do_allocate = !xact_builtin_type || xact.allocate() val amoalu = Module(new AMOALU) amoalu.io.addr := xact.addr() @@ -921,7 +921,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St is(s_idle) { io.inner.acquire.ready := Bool(true) when( io.inner.acquire.valid ) { - xact_uncached := cacq.payload.uncached + xact_builtin_type := cacq.payload.builtin_type xact_a_type := cacq.payload.a_type xact_addr_block := cacq.payload.addr_block xact_addr_beat := cacq.payload.addr_beat @@ -947,9 +947,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val _needs_writeback = !_tag_match && co.isValid(coh) //TODO: dirty bit val _needs_probes = _tag_match && co.requiresProbes(xact, coh) val _is_hit = _tag_match && co.isHit(xact, coh) - val full_block = !xact.uncached || - xact.is(Acquire.uncachedReadBlock) || - xact.is(Acquire.uncachedWriteBlock) + val full_block = !xact.builtin_type || + xact.hasMultibeatData() || + cgnt.payload.hasMultibeatData() read_data_cnt := Mux(full_block, UInt(0), xact_addr_beat) read_data_max := Mux(full_block, UInt(refillCycles-1), xact_addr_beat) write_data_cnt := Mux(full_block || !_is_hit, UInt(0), xact_addr_beat) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 6367b44d..201e67ab 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -174,7 +174,7 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { ClientMetadata(Mux(cmd === M_FLUSH, clientInvalid, meta.state)) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = - ClientMetadata(Mux(incoming.uncached, clientInvalid, clientValid)) + ClientMetadata(Mux(incoming.builtin_type, clientInvalid, clientValid)) def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = ClientMetadata(Mux(incoming.p_type === probeInvalidate, @@ -189,7 +189,7 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { } def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.uncached, + Mux(g.builtin_type, ManagerMetadata(managerValid, meta.sharers), ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) @@ -218,27 +218,27 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.uncached, Grant.getGrantTypeForUncached(a), grantExclusive) + Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), grantExclusive) def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.uncached, + Mux(a.builtin_type, MuxLookup(a.a_type, probeCopy, Array( Acquire.uncachedReadBlock -> probeCopy, Acquire.uncachedWriteBlock -> probeInvalidate, Acquire.uncachedRead -> probeCopy, Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate - )), probeInvalidate) + Acquire.uncachedAtomic -> probeInvalidate)), + probeInvalidate) def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = MuxLookup(cmd, probeCopy, Array( M_FLUSH -> probeInvalidate)) def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), Bool(true)) def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) def requiresProbes(a: Acquire, meta: ManagerMetadata) = !dir.none(meta.sharers) @@ -271,7 +271,7 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isValid (meta: ManagerMetadata) = meta.state != managerInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = - (isRead(cmd) && outstanding.uncached) || + (isRead(cmd) && outstanding.builtin_type) || (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusiveDirty)) def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = @@ -286,7 +286,7 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - Mux(incoming.uncached, clientInvalid, + Mux(incoming.builtin_type, clientInvalid, Mux(outstanding.a_type === acquireExclusiveDirty, clientExclusiveDirty, clientExclusiveClean))) @@ -306,7 +306,7 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { } def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.uncached, + Mux(g.builtin_type, ManagerMetadata(managerValid, meta.sharers), ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) @@ -338,10 +338,10 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.uncached, Grant.getGrantTypeForUncached(a), grantExclusive) + Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), grantExclusive) def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.uncached, + Mux(a.builtin_type, MuxLookup(a.a_type, probeCopy, Array( Acquire.uncachedReadBlock -> probeCopy, Acquire.uncachedWriteBlock -> probeInvalidate, @@ -356,15 +356,15 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { M_PRODUCE -> probeDowngrade)) def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), Bool(true)) def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) def requiresProbes(a: Acquire, meta: ManagerMetadata) = Mux(dir.none(meta.sharers), Bool(false), Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive - Mux(a.uncached, a.hasData(), Bool(true)))) + Mux(a.builtin_type, a.hasData(), Bool(true)))) def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) } @@ -396,7 +396,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isValid(meta: ManagerMetadata) = meta.state != managerInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = - (isRead(cmd) && outstanding.uncached) || + (isRead(cmd) && outstanding.builtin_type) || (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusive)) def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = @@ -413,7 +413,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - Mux(incoming.uncached, clientInvalid, + Mux(incoming.builtin_type, clientInvalid, MuxLookup(incoming.g_type, clientInvalid, Array( grantShared -> clientShared, grantExclusive -> clientExclusiveDirty, @@ -435,7 +435,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { } def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.uncached, + Mux(g.builtin_type, ManagerMetadata(managerValid, meta.sharers), ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) @@ -467,13 +467,13 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.uncached, Grant.getGrantTypeForUncached(a), + Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), Mux(a.a_type === acquireShared, Mux(!dir.none(meta.sharers), grantShared, grantExclusive), grantExclusive)) def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.uncached, + Mux(a.builtin_type, MuxLookup(a.a_type, probeCopy, Array( Acquire.uncachedReadBlock -> probeCopy, Acquire.uncachedWriteBlock -> probeInvalidate, @@ -490,15 +490,15 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { M_PRODUCE -> probeDowngrade)) def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), Bool(true)) def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) def requiresProbes(a: Acquire, meta: ManagerMetadata) = Mux(dir.none(meta.sharers), Bool(false), Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive - Mux(a.uncached, a.hasData(), a.a_type != acquireShared))) + Mux(a.builtin_type, a.hasData(), a.a_type != acquireShared))) def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) } @@ -530,7 +530,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isValid (meta: ManagerMetadata) = meta.state != managerInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = - (isRead(cmd) && outstanding.uncached) || + (isRead(cmd) && outstanding.builtin_type) || (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusive)) def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = @@ -548,7 +548,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - Mux(incoming.uncached, clientInvalid, + Mux(incoming.builtin_type, clientInvalid, MuxLookup(incoming.g_type, clientInvalid, Array( grantShared -> clientShared, grantExclusive -> Mux(outstanding.a_type === acquireExclusive, @@ -571,7 +571,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { } def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.uncached, + Mux(g.builtin_type, ManagerMetadata(managerValid, meta.sharers), ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) @@ -603,13 +603,13 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.uncached, Grant.getGrantTypeForUncached(a), + Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), Mux(a.a_type === acquireShared, Mux(!dir.none(meta.sharers), grantShared, grantExclusive), grantExclusive)) def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.uncached, + Mux(a.builtin_type, MuxLookup(a.a_type, probeCopy, Array( Acquire.uncachedReadBlock -> probeCopy, Acquire.uncachedWriteBlock -> probeInvalidate, @@ -626,15 +626,15 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { M_PRODUCE -> probeDowngrade)) def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), Bool(true)) def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) def requiresProbes(a: Acquire, meta: ManagerMetadata) = Mux(dir.none(meta.sharers), Bool(false), Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive - Mux(a.uncached, a.hasData(), a.a_type != acquireShared))) + Mux(a.builtin_type, a.hasData(), a.a_type != acquireShared))) def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) } @@ -666,7 +666,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d def isValid (meta: ManagerMetadata) = meta.state != managerInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = - (isRead(cmd) && outstanding.uncached) || + (isRead(cmd) && outstanding.builtin_type) || (isWriteIntent(cmd) && !Vec(acquireExclusive, acquireInvalidateOthers).contains(outstanding.a_type)) def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = @@ -690,7 +690,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - Mux(incoming.uncached, clientInvalid, + Mux(incoming.builtin_type, clientInvalid, MuxLookup(incoming.g_type, clientInvalid, Array( grantShared -> clientShared, grantExclusive -> MuxLookup(outstanding.a_type, clientExclusiveDirty, Array( @@ -725,7 +725,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d } def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.uncached, + Mux(g.builtin_type, ManagerMetadata(managerValid, meta.sharers), ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) @@ -774,14 +774,14 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.uncached, Grant.getGrantTypeForUncached(a), + Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), MuxLookup(a.a_type, grantShared, Array( acquireShared -> Mux(!dir.none(meta.sharers), grantShared, grantExclusive), acquireExclusive -> grantExclusive, acquireInvalidateOthers -> grantExclusiveAck))) //TODO: add this to MESI for broadcast? def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.uncached, + Mux(a.builtin_type, MuxLookup(a.a_type, probeCopy, Array( Acquire.uncachedReadBlock -> probeCopy, Acquire.uncachedWriteBlock -> probeInvalidate, @@ -799,15 +799,15 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d M_PRODUCE -> probeDowngrade)) def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), acq.a_type != acquireInvalidateOthers) + Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), acq.a_type != acquireInvalidateOthers) def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) def requiresProbes(a: Acquire, meta: ManagerMetadata) = Mux(dir.none(meta.sharers), Bool(false), Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive - Mux(a.uncached, a.hasData(), a.a_type != acquireShared))) + Mux(a.builtin_type, a.hasData(), a.a_type != acquireShared))) def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) } diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index e7291222..e58dd08d 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -325,7 +325,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { when(tl_wrap_in) { active_in := Bool(false) } } - io.uncached.grant.bits.payload := Grant(uncached = Bool(true), + io.uncached.grant.bits.payload := Grant(builtin_type = Bool(true), g_type = Grant.uncachedReadBlock, client_xact_id = tag_in, manager_xact_id = UInt(0), diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 32ccac87..3fc0e4e7 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -76,11 +76,11 @@ class Acquire extends ClientToManagerChannel with HasClientTransactionId with HasTileLinkData { // Actual bundle fields - val uncached = Bool() + val builtin_type = Bool() val a_type = UInt(width = max(log2Up(Acquire.nBuiltinAcquireTypes), co.acquireTypeWidth)) val subblock = Bits(width = tlSubblockUnionBits) - // Utility funcs for accessing uncached/subblock union + // Utility funcs for accessing subblock union val opSizeOff = tlByteAddrBits + 1 val opCodeOff = tlUncachedOperandSizeBits + opSizeOff val opMSB = tlAtomicOpcodeBits + opCodeOff @@ -94,13 +94,13 @@ class Acquire extends ClientToManagerChannel // Other helper funcs def is(t: UInt) = a_type === t - def hasData(dummy: Int = 0): Bool = uncached && Acquire.typesWithData.contains(a_type) + def hasData(dummy: Int = 0): Bool = builtin_type && Acquire.typesWithData.contains(a_type) - def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && uncached && + def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && builtin_type && Acquire.typesWithMultibeatData.contains(a_type) //TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache: - def requiresSelfProbe(dummy: Int = 0) = uncached && Acquire.requiresSelfProbe(a_type) + def requiresSelfProbe(dummy: Int = 0) = builtin_type && Acquire.requiresSelfProbe(a_type) def makeProbe(meta: ManagerMetadata = co.managerMetadataOnFlush): Probe = Probe(co.getProbeType(this, meta), this.addr_block) @@ -111,7 +111,7 @@ class Acquire extends ClientToManagerChannel addr_beat: UInt = UInt(0), data: UInt = UInt(0)): Grant = { Grant( - uncached = this.uncached, + builtin_type = this.builtin_type, g_type = co.getGrantType(this, meta), client_xact_id = this.client_xact_id, manager_xact_id = manager_xact_id, @@ -140,7 +140,7 @@ object Acquire { // Most generic constructor def apply( - uncached: Bool, + builtin_type: Bool, a_type: Bits, client_xact_id: UInt, addr_block: UInt, @@ -148,7 +148,7 @@ object Acquire { data: UInt = UInt(0), subblock: UInt = UInt(0)): Acquire = { val acq = new Acquire - acq.uncached := uncached + acq.builtin_type := builtin_type acq.a_type := a_type acq.client_xact_id := client_xact_id acq.addr_block := addr_block @@ -160,7 +160,7 @@ object Acquire { // For cached types def apply(a_type: Bits, client_xact_id: UInt, addr_block: UInt): Acquire = { apply( - uncached = Bool(false), + builtin_type = Bool(false), a_type = a_type, client_xact_id = client_xact_id, addr_block = addr_block) @@ -181,7 +181,7 @@ object UncachedRead { addr_beat: UInt, alloc: Bool = Bool(true)): Acquire = { Acquire( - uncached = Bool(true), + builtin_type = Bool(true), a_type = Acquire.uncachedRead, client_xact_id = client_xact_id, addr_block = addr_block, @@ -197,7 +197,7 @@ object UncachedReadBlock { addr_block: UInt, alloc: Bool = Bool(true)): Acquire = { Acquire( - uncached = Bool(true), + builtin_type = Bool(true), a_type = Acquire.uncachedReadBlock, client_xact_id = client_xact_id, addr_block = addr_block, @@ -214,7 +214,7 @@ object UncachedWrite { write_mask: UInt = Acquire.fullWriteMask, alloc: Bool = Bool(true)): Acquire = { Acquire( - uncached = Bool(true), + builtin_type = Bool(true), a_type = Acquire.uncachedWrite, addr_block = addr_block, addr_beat = addr_beat, @@ -233,7 +233,7 @@ object UncachedWriteBlock { data: UInt, alloc: Bool = Bool(true)): Acquire = { Acquire( - uncached = Bool(true), + builtin_type = Bool(true), a_type = Acquire.uncachedWriteBlock, client_xact_id = client_xact_id, addr_block = addr_block, @@ -253,7 +253,7 @@ object UncachedAtomic { operand_size: UInt, data: UInt): Acquire = { Acquire( - uncached = Bool(true), + builtin_type = Bool(true), a_type = Acquire.uncachedAtomic, client_xact_id = client_xact_id, addr_block = addr_block, @@ -308,6 +308,7 @@ class Release extends ClientToManagerChannel // Helper funcs def is(t: UInt) = r_type === t def hasData(dummy: Int = 0) = co.releaseTypesWithData.contains(r_type) + //TODO: Assumes all releases write back full cache blocks: def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) && co.releaseTypesWithData.contains(r_type) def isVoluntary(dummy: Int = 0) = voluntary def requiresAck(dummy: Int = 0) = !Bool(networkPreservesPointToPointOrdering) @@ -317,7 +318,7 @@ class Release extends ClientToManagerChannel meta: ManagerMetadata = co.managerMetadataOnFlush): Grant = { Grant( g_type = Grant.voluntaryAck, - uncached = Bool(true), // Grant.voluntaryAck is built-in type + builtin_type = Bool(true), // Grant.voluntaryAck is built-in type client_xact_id = this.client_xact_id, manager_xact_id = manager_xact_id ) @@ -363,19 +364,19 @@ class Grant extends ManagerToClientChannel with HasTileLinkData with HasClientTransactionId with HasManagerTransactionId { - val uncached = Bool() + val builtin_type = Bool() val g_type = UInt(width = max(log2Up(Grant.nBuiltinGrantTypes), co.grantTypeWidth)) // Helper funcs def is(t: UInt) = g_type === t - def hasData(dummy: Int = 0): Bool = Mux(uncached, + def hasData(dummy: Int = 0): Bool = Mux(builtin_type, Grant.typesWithData.contains(g_type), co.grantTypesWithData.contains(g_type)) def hasMultibeatData(dummy: Int = 0): Bool = - Bool(tlDataBeats > 1) && Mux(uncached, + Bool(tlDataBeats > 1) && Mux(builtin_type, Grant.typesWithMultibeatData.contains(g_type), co.grantTypesWithData.contains(g_type)) - def isVoluntary(dummy: Int = 0): Bool = uncached && (g_type === Grant.voluntaryAck) + def isVoluntary(dummy: Int = 0): Bool = builtin_type && (g_type === Grant.voluntaryAck) def requiresAck(dummy: Int = 0): Bool = !Bool(networkPreservesPointToPointOrdering) && !isVoluntary() def makeFinish(dummy: Int = 0): Finish = { val f = new Finish @@ -396,14 +397,14 @@ object Grant { def typesWithMultibeatData= Vec(uncachedReadBlock) def apply( - uncached: Bool, + builtin_type: Bool, g_type: UInt, client_xact_id: UInt, manager_xact_id: UInt, addr_beat: UInt = UInt(0), data: UInt = UInt(0)): Grant = { val gnt = new Grant - gnt.uncached := uncached + gnt.builtin_type := builtin_type gnt.g_type := g_type gnt.client_xact_id := client_xact_id gnt.manager_xact_id := manager_xact_id @@ -452,9 +453,9 @@ class TileLinkIOWrapper extends TLModule { io.out.release.valid := Bool(false) } object TileLinkIOWrapper { - def apply[T <: Data](uncached: UncachedTileLinkIO) = { + def apply[T <: Data](utl: UncachedTileLinkIO) = { val conv = Module(new TileLinkIOWrapper) - conv.io.in <> uncached + conv.io.in <> utl conv.io.out } } diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index c05aafee..cc4755f0 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -260,7 +260,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val state = Reg(init=s_idle) val xact_src = Reg(io.inner.acquire.bits.header.src.clone) - val xact_uncached = Reg(io.inner.acquire.bits.payload.uncached.clone) + val xact_builtin_type = Reg(io.inner.acquire.bits.payload.builtin_type.clone) val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone) val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone) val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone) @@ -268,7 +268,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone) val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } val xact = Acquire( - uncached = xact_uncached, + builtin_type = xact_builtin_type, a_type = xact_a_type, client_xact_id = xact_client_xact_id, addr_block = xact_addr_block, @@ -355,7 +355,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val needs_outer_write = cacq.payload.hasData() val needs_outer_read = co.requiresOuterRead(cacq.payload, co.managerMetadataOnFlush) when(io.inner.acquire.valid) { - xact_uncached := cacq.payload.uncached + xact_builtin_type := cacq.payload.builtin_type xact_a_type := cacq.payload.a_type xact_addr_block := cacq.payload.addr_block xact_addr_beat := cacq.payload.addr_beat From 57340be72b7bb9dcdd8f38b759df4044a77812da Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 2 Feb 2015 01:11:13 -0800 Subject: [PATCH 283/374] doc update --- uncore/doc/TileLink0.3.1Specification.pdf | Bin 0 -> 172795 bytes uncore/doc/TileLink3.0Specification.pdf | Bin 123869 -> 0 bytes 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 uncore/doc/TileLink0.3.1Specification.pdf delete mode 100644 uncore/doc/TileLink3.0Specification.pdf diff --git a/uncore/doc/TileLink0.3.1Specification.pdf b/uncore/doc/TileLink0.3.1Specification.pdf new file mode 100644 index 0000000000000000000000000000000000000000..23666814d9e8738924e59e5f9f8872e7fa58d725 GIT binary patch literal 172795 zcmY!laBVf>Z^4=fsl4ocwey{eZ;u)M5og z1p|d3E;~D})VvgsIwOz(hym9CQU_99l$uupRjwbDTAW{6l$=_uU}K{nP?Vn>oLW+> z5Un5J;-;VrnLDj1rW z=$V@;7#moEC___oJyVdeAXh@d(=oXuGe3_@KUhIOG{}<+L?~!vl#~=$>Fbx5m+PhH z=cnhS>Lusr>X#Pf*caM@ay;6&dQ{$adOZ5s;($tC*ZC%ug^Gd304b@7E(`_By0<)a`+(UiJ z3zCv@ibMR0g36s7iZ zbN|`uq ze;=ZMczt~Reg7Yk{8umEAC=ix_y4#5e<`PT|G$0xeScp3F@w0mV>=tCC^|uREKp}NcR=@SM39KHv>kZv+I%EkctlgWWY?;ID%XrCn zqV?|W7dY03MBj4=5>_zY>-I)YS940Z#9M|=CY5X9xryrAxg?iPiBI10ZiC*_U#C=4 z@)us}xbsLaMFE#RjMEK zYC7DQc5RubkzX!zV3hZ-Fwalc9anAz#V`J{rP*Vx?v{Y|Wi|>y+1k2O%2d3jN7ZaK ze9g9K$~NKrBAaW0UePa>_?0iTi<6Y8bC`0ePjjP2lv?hJBh4`@y&EmeKizO#qUU-m zu=nP}m@b!hXU;}zYz;quHI`$W`jgs)?wQl2E{0{SJmhe?tS)R<$(t8}Y3{DSXJ6d! zA*wrF^3}CxNq$Qf6~9pkx>#Jjl~1(RVT#z~vaca!0-6`SHgzn`RdCe{@f6#7R<=8& z#Ar=#=<|b0TVJnT{H^bv)vtt!Yor79mXuB@J7T4^vzmLZMgPdGRx#Qx$bQ3$*+QpGpR-L;ndWs$G;w-Pl-8U0m(m?U7u~pfWf%R? zddm5Dn{&o$m(w9JHvAheMesLVSa0sNWY**(tYS-Rg{^9u{^;5R4R zscnvD{dF#FQ%$O$cTrKiO?!^Qt!-1qC+>OgG4E7J#zLdE!@EKfE_}Z7M6)_xkGnGy-oCV%*ClGHC->KF?>#CVS3OeJblxnSS9rv&&keh3`|Bf1NDbquybp(p1 z-8^`w+;zpFdrPdQ?G07hbyav@O4oY1eOJ#XJKuk$9;>!;c3FeX%vAQA_KYv5`EI;> zwr0Al!Qbs7MwVUd9lP?ARL(Z8G_buSa(h?1tiYp0z0Gg8tdK65bmnnMnae+`lY3P1 z-9;W>6<+f1(=?6Zy>%16AI#Xd^Vg@gC+;#wO?(?KUER-;Ri$Nk-h07i6V@HqBcrzc z57R$>iZS!4(VK0%Z(dHlk-J>fPfoh*Howe=oBTSFk$Me1$(Ju>&MmR#>C!eAsO(~v zz2H)~bZ=(ar>MDn<-+Yf%9gX+#J&sd+Wc~gmg{o%X-nqpQQNp;vqO~3v3=>4#VR{~ zv$t4r>Uzk03NVXW+W(@!-ZaUJhdujgpLOo?WplT_`X*VS$8$6~V?r30&ZRm2w?3^o zDOk+6CE#7C@w-no!Zz`ra=04qJf45}LQ>h^-amVv|87`ez31rHny(EZPxH@wt=ZbR zqM6&j;oA<~FN!ygeZJb}vdeC!ks` zH`evW0&kfY1@k0#Zo9E<^OMvR868tS<;{tlv(C?H)i`JDkz^eAMxAfh^q&ilI3DUN zcyq7qS?aq}%iD#XW&C}0qN65T==qHaIWv{nek|OQra1l2;x+b6m;bLntWY}Tk82%! zt;3XM?vs=)xvV>?{-|08UD)jCaWBxtm??INbjO}4UMk{imh!snzp_01eaY6`h$%nY zKW};bY$P{`QN2GMEtl;E7>W$dTZDFwe0>B3b6EMQ6xoY3F0yHU8zX>$h6$ zny;~nb8~tB9Yf7G*5?-9`l6C6dE9c!oYPCVHlKO7!in##=*zmY77xdZs{?di>xvv- zy)@a`sjxUJr7ykc+nhi(p~buX)V6(jIme}_^2D*DZxj_b8y=VF(|Y3dnnS=-*EOU0 z#Mf^=e!g!ktGNR^F0ph>5r}LJJXEm8$!#P5#Wx?8PS6TCFn3O?m%~iIuU96O%)1yE8DhbX~BDMuO~~FzFZNb_}^^%tU0#g8Q$DS7q@O%x@_NM?Oci2AeYmeN|x^7 z9%{2+CoZ&3;Sl(k6jh}%d3M+Fm|cn|3M#?_6-!^f+*7TpJ6X)iQKVd5_sThL-wECa zzh}2Sd+F5scIv4TmynlD9!p*LIb#JxO*6J_T_XL=^zw&1rCoF4T^mlcp7{4&PWIk4 zuh&l=t4&g0GHK4^%JvhwlS38yZLNNVoDA1c-5uRF!~Lt?YA(b13R#Lm#*$B88myZ* z<1wew_d6#teLn9v|77ir7fk}(o>Q|q`Kmq6Et9tFwOsZpXIG$)wV#0I(Xfn#4+W!V zMY#n9W!i78NLv@WNM-4p4f`EMGfekRkYwI+*!=&*j)gN@mn~d#qmDT+F*$1EUuL1y z9RF*36t~qoglw7iitE3*z*OB+GXL@gv=Vm2zG;toDKq`{(Y$E^0sK2--mrI9t?78J zS+V%3^I5j#b7XWx>b6Z5i)9uI|9ggQQO(b7_e9MFran8T?U<2o)mrDuqV|8{rx?LT z%?6R@TG<31_Qze{<$|vEs}fhoqTq2J&;y4bVHcWoqBvN=7AfUeCpKmmlWM zUVWt4n&DFD`z-<8a{sIMGtDsXs5#WQ-~Qa?RYiVVo=)TcJ3V^df+q}tOB9q&T(org z@Xh%=`}d{s-1+jME4MWmX>vcmIj41j&WQ;()m1Lf7)J*I@9)?9R(GA#eB7WQ`z@q$_7m1~by6CF#+!G`$EuWkHZQTcaXnITv*GhX0sCE_ zWV~z@_Ipk!WKCJN?ZEBjPxAFwHRZ&67K8sD-iy~iq zTYGP6IP+2cwbfqj4GK>_Yk0l4PzreU&Z$%8c7+A^sf}kjrufxA66bWX7UFBYu)-_v z!HnmO6a35~`=-RUH$A!$&bi!KQDkz-r>ow#Q&k?v9$_&S^S`up@)GwiIqq-6ccgOd z;!8Shqwjk1zvsO|yF44$Q+t#piqGrb5$rj6N9;Yvu`jyOGAu=MZZF@f-`dO|I6F^r zV|#gw^rPcZdv9|YGRDrkoanv!VQAD>9)aU53(edA%Lq@gl(pR6XYj(pb(6q`iLnzR zmUO61w4VGtZL4RH%-@8&>+6nObiSrj^muaO%tz{Wl4ADrf22gHv1jV~{jgQg(r~CT zI=%5i`?(IES3C5VI7w98v^actQf|xbs1H*TFFScnxnVCiZJW~fSbrU>w#?OgQam8z1VWZF0LcIvc8EA5Y(Pgl?AuDmyIfiL%?$(3`$GCvx}_%SNQ|9DumNc`qOr!+;b z3G-Ds;(jC-mQ67Z;%v{;4xh8=rsZO9?}e>?0#Db5FA*v#4dHs|{^siCcXv*9>iwLs z#mi$t`!x5D8IFG(x6eqn6sT{RF`3=y@Kmo2JH4kh2Xm@zRc)A3Ge_{}4cW&}|IJd; z`rwtFan4XHB_>Pe;$qc_`K9k7+cjREs+n7Rnjtr7out3j`+O0 zwK~QBvDRHNYMunlnS8@TFv)6beAP|C zeZLm|wX$TxUd?X}O^Esq;CwoO>@nsd>H2@BkN+wcB3+b?>nF!H}c%EhKsO*PBU8V+>|Q}*|V zfBpO2XsO`izWA!66DbgQ*>fVE+%{gdg30QGyntvxib}kP zzMMelx&A#TJ%0C<$n$+NUiwC1RghTYHO57!m%LiF?Vj0Mt>2ecZM)|Xa<(kSQ;?l| zWw@w;vn#ivW5VRw6BJ%2Ry;X$+2!fQ zZ5P`2aF&0zh|szi!!@0hjuy*s&! zPaCq!eSh1^b#Ld#XL+k*IxD8}w{P-rx7UoCa(?!Dou;!^pKgc!ouS(hbm&xug|$|4 zYv4I2(S_bO%yqRVST1s4NQvLGH1=!qwlZ;>h?46riuyvA5*MYg7D;OSwR+pXr1f2g zS$5f%);{aICVvZ5T{w$)!x_Fb{LzU`z9g>qca!SgPf7=c9b8$e^p+fAT;k{SRj*;z ziGylomg!PQLl&yMXL2wMm0Nyss&&)8ISN?|+g7Z1e}3b7+lu!L7d`v6dy~CqeO6-X z?(+5M$*nqdpmUeWdg%Q(eRi^@ny=-k4-JT zxinQFzGOyuR_TiC5)sCm64sUGqh%%@ZOtzRK8~WqV;|LE^U!nM_n-f>$ zO*k2BVa6dDb*FMNugg(`@OwvY8d++qw#<#N)liry*|2I>^syb?23ds}507}hs$~y+ zte4@=-tg+8Ln60cZ@Rr!zL8$~y(^+;nyjW>{Or5w$ihp;qW9{)ny{9}H$}usvE|EO zpYEIEE_trlD zOnbqp6kEFPilsw{p=i^bqB+iUg%u?^1+E@1T$hWLv3R(e>i zn(QYqGs4j`Z0(w{V;i-JPEIEz8iEY_9zFvys%u%_V4pmYlU5w zyRO?N{M+Taor;*No?3kC8%^`_qA$tLTuGNwOQgi7iaH9Vi@3YEym)%uPT^5)OWkKJ z^^R4`ytXsv%DtGjo~QThw6JwNXPJfezwXtLpYfvV!hauS*XnT7nQO0~U1LzaJuY7% z>~*Pe_ThVLuk&}@ta^K)XnUObX1}+2d-FEE%WjTXd;P3Vd-e7>>nSm7uS+Ysz23U{ z%-yQHXN=5vCOok;IIGikIA{ITnFg!x&7ad08T9Lk$eVf1<$ec4T11Y$in);yxK|?l zfAiVtQ<|bYraJS!i`np%q4uiH>|2ga$6f`^-mW;W+kVmE`fs-k15c;vEvr^75wMPU z(AxTnL$FqC^OrX(WWI;aSGaX|pVkL+ov_zieRaQwE_W_2y<8IUlWBMSgLiwjh4$5! z)xF#vts<~*>Z8D-^0<7D5Vyu1hxGq0+f<-^%qG+?Y2|n0eZ5QX`ad_@`L^zu`BtOv zzf14t{o2vJ@Qv^%y|zuhsq?na&Y5ieQO|DA>qJrKx!fz+-tF79Ecb9=(#}7HyI(WB zI`nVHU5QB#HnIE1Ta-6`QkuQ{{kNRiyIEQF9`1-NHNEn8S6;cp0yeez=Brn=IsGt* z-M3{m)5(Rb`4=DW$Vgc%{#-BX{>iD6XLEHupS7Fsy0>h5+;Y8Vzr;HCD&{?KyWDto zr_}mo&AcvEJ2?1QruknDcyVT{6u&tjxZQ03C zUD@x+TQ2uB{#dxP#&Keu#hLD|k1vd})H@Cty+59Dxok^=--kCVMa*;NM1;J0BrAJk zF6YNz9(}n>q87ED3toTkk% zd39r>9_LxnmN)T+kp0)&SCljy79VYU zdU?arD~mP@@JDAQFLyO}`0{n_Oy2HKKW@*Ou)FcZ1*O=zyM4o+iTBc)tT;k_~cKkxHj`DA=J_jXV9@qHl1+2=n;kBg_&%5ll&f9t|F zMKdnYDXwbQaE{z}xQSObqD0vI)S^dyu0KD&`25V|WV7q9&u`Byx$$Jt`Q1mJ>9%~@ zKG8P&Wb)h~?|BRY790XEw=Z&E)cfiM(|3;=b|V$Bzt`@sU!=a<{ZZ`2lq@~}6f4Iq zr|W%>Y&y&2&KmP2#XR}SYW-V^+hc_{%0%50_4SiJI;-Pp!0q*&5qu|hs3qOJb=K6M zsr%I;k1WMb)?!A#tk63Rla^1Aed>0{pvvk?e5;s-agoL8#S1N7CY#F^m|kXFl+3SV zyjS`BRUc#5{)jDit{fJ=tl7HBt=Mu)0^5zu8>wZ?k{Wz(WCVPked4QmD{&@tlezDJ`ud2Ht;Uw5HLrAbZ&!@-{q}91p>3Ax;Wu{OCE@8ew=8(dR%^{C zc4S7X)u~H6=BuT-M@H=xj$O9$i5Yjv1?z|pcP_XU|G2;7`LiR7e(4-~b-29jh^={P zyvH_4lPUYIzPVe$n``;xmTaoefr!{lcnxHeec?UA4j~3&PO`g@x^{S*SGVtvZ=en zjQZ2D-^{k&k!jjlE#1MjJ>xW!OYiPOPQ_Ngj_^MJv0|pKLfE zx30Re>+HcRoQocHRXp91bGzvt|GJoB2Za}tWA1KkyzzdEt`Rqb=C=ouA$r|TeW#~A zs-88~ncH7XVqRR)YKvQGIgZ7vE8ZTL4D6LvQT~({J7M*+GYeTc?+AB$i7eQqwfvoi zEdL&-zSGZ6@7{R$*}L5v4O0~`)-ZgcFFe|TgO%K_oZ=UY=s z4sve4A;VZ;&{BEg+p_3A&lGRZ&U>NjeCu4KM(j!Z-$9QnUb0+V`#M-}$)g$COE#^a zRqnC0XZu{oOvm7-$5_j5J8-zb{@}~B*cVyP?Ho$hABl{7f!_v>QobIaDJai*W&TX@*j ze6{?}`?Kv&Co4DVS2Wf9+M&NZF~{=pW{Ek1uhM=hf3KdeQ04bdcHs=6sI8Ie9bSi@ zXp0zCcF*I^)49Ll#4OD{YZh<3^GxrO!A~1SBO#~jJnfcc-AqkPHS*^U7V#h4`nA*j z&Z6UY1qIX89`Bwm-SI>wX}7S8^n=Yt^e)BAbrRnaG^{{B)Y`qt9XdwUY{&yXGEB z6fKEbyq16ZjVDD>%JZIum}Z?1wM%~;d@$Y9PayV9i%(?vx1Kd_2U&!FuPHjEw9Bj~ zSnTVT#7mEldsy_@`8+-^du_#r&7}*!tmZg>)0p*zrF;28#;88!_3G!BOB$1tITdYh5AVIo{ZIqcix;MIg#aMo5wq+eD^!|O7^y; zS#C*~@v-3U{H|?LR~Fwia+=Mo&Yg2KW^(%CbO|~A+j%=3&cESYB=_R|x0k#wqSklg zw%t7<@H(rzP|Qi#+4g;W=&}~8jc;O2-Z(A)ylDD8{XdVV**-qFcy80~g5Ml9FVbxJ)F*g6b{2h;2zE3*B>tfn9WsBAe znW$40Q;mMHEqd;B_Vup%4JZ62%6)jZ`SK$Bns(3df&{su(8FK$*xYOv`?|5mZ(hy~ z)&(;8HJr?vD|lUMIg=NiRj{miSm-M;am&R;&rT%81i$X<`7!&2$71UzNjglwqrW-5 z-4@K>A+_bm<)t#`63+=0>SblUJs)#!6>F8V`lJ;$qRmILJEqxKuA5Tp5R%M$?Ej)| z>CAz9150f$+9-HMP7GGwm@A;A*S2z1CVS?;YZ_DTEvP>*p|5_~%bc3~CEprm_@*CT z!&7HImv^(zpSutNKHc)mH<^=6=kjiT^*zr@I?eU{?w2_;SWo=WF{@Y2m@)O-k1bWd z*`EBH^W5Nd)5qM8UGIPAC}f#^`&uhM`!8o@_LnI)w}TKU;g^2Qw|QgA7saHu`)o?=Vn?!Xw4Jq*t89PyNLtsZ(p^!w{`0YG z@pY;vFW)X{TXHK zRxaQ2l`}hCam#$g$=T6bNpDswOwQiCu4Oet_{8iecAkUMiQDUEm)`LEzH0OHPp@A$ zzitd%;lDy#cwzXD4{R+-cgyC){QS`L=2oYxZE(VwxZb0JVUDez&gk)<{(3Tk-FvT{ zQnP^4{bIXv(JPnknh{O|n|@MO*rH!oleRfmW7h zjI#dmY429r)}b~pz)vS@!||^leP@du$&883{LET->&S)u>?;3Gh9~j#2kUH#HxB+W ztNf|UoTt(+g5)yR7M0jv{rf#=rbd?3jLr6E56y}=&Yiq?na*DelUA9QJ&h~cnT6xxKul!!BUXCdEKu$;(!)m)EE| zaXRwrJ-=}Br0AJ%^9=7V+o46vjD%hrL|lHB*S26kqvB+R4KE}ON=*;mGCFj< zY$s2hediKx-na9<%@eeDXH;r8nxw~}vx~dfOJ~!M zNjB5M-AuASvohv8`sBRl*c|C*(_ixSJy(eCD$~E!FV5NjyJ4HMY?>i|!~LDhXFB!r zh6mbjSP`wVe*G2e2)8HeujC6D1sbaTG~IB=Pf_ZnaCF{s0p0cU=47hP59V3@Fm2Y! zbS~NH{ibJ)m_uf6>^D7orRY@R)Xg`o4;kxN2b}A>7v8d0xk6aIqpof4YN4}?naQDx zauq)9dR1t<;MT5&{K+4fSLtT%Q~J>GilbpsY>Mgj(koLo{1!5N&0Nsw^Mv88?c?J1 zlb2b({Q7+Sw7FMEwO8!ou3X7SGq)|Ca{Ze69sZcYr#d0OR_)l-FZqsLE+VaX{`>aB z8ZVX!CfP}8*|<;HXm6u-?_7N zXN7yqnP1uq-W;;)o^d+VK>5tVR;HH9rWfxjqBwjdz6q|JEOw|QVYb+rr>~apT48=- z!#uX!pbIkR;~mO3JV-9qv_4|C>8n^ z^53uipZxsTDXyT!SkUz~u+=%BMN-h!IgoWn@MTn7`p%9DhFto|i6GL)S;2ry-^U9~ zhJ#379|aRG{a{ChlA_X7E`29o1%1E#qTIwB(4rui@u1aKD8_@9Vj(S)0@-Q6rSFtj zoC=ar&<`m}O)5>!NG56KVB%S_HsNmVei1PLL#1hm+TB$qfAWhUnMh7j!vW6%mQR96^- z7QLaYf&;BHBgqv=;i2o4pOZqob3ieR<{VILpg0G#){P|RxD}eE&iHSnAk-3GQv5^5NW(*We4b1e6P0T?n=0K*InwsmG8JdG4(m=t) z!a&c`)Lg;T!a&c|%*-TK!OTJ-+SJHY&)CwyOaY|U(8$74!Nk&3&k&>uE@En8s%K(o zV5DGdW~pZeQe$dhsAp+vY^Y#rVy0(oVP>viWMQgjXle#>hN*#xp0TBgse+NEiJqmA zp{0U_v4Ng}xdoDPQ)4qdGb2MogtZ0=#ugTOX6A;L3MK|7dgd0UMheE3W_l(TCMIxe zVq-xe1}z)Fv567si0I}5FWYp>%t7nF#cz>7b1Ell++jm=FJbdAi6^o$J56$}k5jPwkQ4UH8nO^oy`%nc1u z{04F_)NhdWjfM(_=0+9ALdKpGmD8XBT3f_z(DSrg)V?ZwJn-@onMtFm{={gy2|qy+wP&%`d-7uXA|V_Zuv$ zJo(CX=G~k53^%4T+%VJ4%8vW>TV*p>LXEV;uk^>K{Ju(6e)4NjT+NUX{rFOwneVmZ zxeQiw;%{;Jl(mGNk7a+|!}`A=g7wDpm2G4^UA`W<=({n zFHe~@;(x9FvwF&0X61wYrJLkx!Wwvp6X)VscK z3o_)MBw*Izp0MYOMvjtdKv%LpyCU=Pb#8w|4*liGPx#HsqJBuRrI5p0Fi9ZKVTNRj zREyEE^Y>h;`|S0DP7+a z>LtF}vxaGk6$*b^?{cT9LfuB8rsap=1`o*--*s9t?=>216i(!JGB$Zw)&4TD-u{v0 zj(_<_bpH1ziLDfHe_S1VLHUocnxdX~cB0T<_Df3Nt~>JUwrc&-jd0Pu^f__$v&0vP zf0HbqzfXM6;hebIKu(QQIYFu~R{0Mv&n5l@5lxAf_k#BtLSoD%uU8v4Jha=F&z5t4 z$MXyGcKq>KHlI;1`R&pJPqkY)w_N!zb4Z(cQ!Jmx^#p-=Dch7*YMegJ@w$~gQS4D& z%hVjcO!F;88xD1!C@;NmFFbo6>t+*9u8DH8TT&UeIdH|_=G~cLKYa&_(9!LQcCUVF zFDq_2c_~KYddz{nR}NLA)jz8~_}$lO&Hk>}Pcz>+UstwKlXc&{u)nS3=ojUB{?<*i z_s4fX-g;s?^TWiP`mTZrXVw2Oy!3g0=iB2CcXd1tY*ZJxCo110^U-&U=sWq_?oID) zbixl*?Bn~qUtpfp2Yah)?;R}~ON_S*ezxsdJYlcXft-U++!8*1a z`^Ap*cP7@!$8yX(lYgo|-44YIjg{Wr_q z;kYcg?%_A5yGD0E)}|knl3V^_`xhO>41VQq^=FrNw7WNF`z7r=UaQqI^M7*8gWdnr zc0H_0*e8A{Jz@41#>yErKK5tWoh6bdA3d)6V_`#l@lU52FZ+HfcSrWjR>+f?z?nSb zxZq*?LndnL|#cjgl@L*cxnO_$z3)YUw@KUVnpeZTrI`P1XvZ!{N( z-TwJ+UE`c&R}JR~7T-%2Q?{~yoy@=Y(cT-+8C|Q4CS)IJ-q3M-*2i+?Gi`~F7S9ki zbDpHeb}AsB&DW~r@g4omc6WlOez1AA-*toJ^WEM#j=vMY=k1<=l6Ae*W~j*tF>{XWpfV znP>j7|B&jDKOQN+ZgXgr{HLRd8t*S_oadZz@>|=Ucnkm}koU=S| znRMVanRUnWELopB{hzR}t>$dXo8(>JoceMerXR2{6kGN{PRrH$*^|6Gmzh_*t~w)I z-#%Nc`C6tR?GPPCjCq*I(UI9H3&u@K|uR$xg=D!oNN(OQnRr zWVk8)wS3~8Eg)_2-{qykrI{6ckD?wq{b$`~Hh;OuvI{3y>^}ML{7bI?*|T;h%vi=% zIkEk81>fV|wcWcOeQHa3`sv0lo46(mk?HmaSr1mcbhufld&Yi#a-Dta56>g#mHrt& zN|rdE9dU2sy~f%{);q-JP5rYc>G{%WzmL3KI76nxQTvwi|4S8nboBChjwKfv$QkSR zA3kkpo&R%YyJh!NgGvPtkuU*2^b#AuhH^bY1^WqZ$0-N^4vyB5LH{Es#IZ%QN9#>yF;Jf@{8T zhWUnt@h1kFvN$q}ck48Mj`?-a(Q%5&rbSm@@+LlXn|xC!vA3sd+fklQSt;@P3vLO^ zhO4M9U&1Fn&n)T5iZ4B9*1WlBnOyYw)A2rS3H!uF#h0EOTJ%U$b!qVFQ|s5Pm=k8Y zx9ZrL$j#4pWIa1KJAU_qb^92_%*=L*n{MBtw=e7DrPR~6!?u){l&yUexAoU6)9m#- zzwIi$e!t-Hui151H$QyVyZuhp?PX`9^Y%YJ7ym!)^|syb{=K#R{_ows-|`g&zn%H* zKfUOi+i&~t$Yg)pzdh;Gzu&yO|NiFf<@by0EB>|q)!+B!ll=cLr&hD`$yL4m@H$_n z^7Gp5_Z!RaSO0J3@2~svaihQ8-#4Eht6vDqE?geoEd2V|#sb})eRmxHd{LdF4kzX$5mzq8ORb>6Dv|nC{tO~R9 z|63Qf@G$d^Z4zEt_>`|te>AEFQPJuqhe^8W4jZ|{?yy^p`Uf7Slo z^{M}-=*#_4dR}osv+sUM&Fjv3`TZ}Cl&hbUi!VLIT#)p*@N?%x^((?3=j;Fb^H%uN z$K?6Bf8XprzoF}Y4!hIybZxuUzjgj@?(u*3@5#6R{eM$#s_U8ce^Dym7rIC9JQu_A z;|W*dzAe1{a&gIb|Elu1d&UMu?hJKlDYCKWJJla5M(~?YytqO1+=2B8!grYVHy?kH zXu&Ss$z*)&=b_=A*8%ko4;lS5(|P1g_d zwm&e~!J|-*)>Ua|QpscCjDQHN57BlQLRYCur(4rhZ`1 zX&1e~_Vxl}>;+Ekh0^<+&c^VrPUMYY&R!@k*Yf&7)Sja;{Ps@rHtv0s`a9-Hct~)n zGYOtN{zT$uOO2PxB1N|(r8K3<=TAI;Vqxj{&uO26n2_Tdl}U3{>RIJ~oYX9uHAgXi z()ttkPh4NpU*o(_HT`7ylj=|9pO}BL&wA$k$^Vo7C*_~!mhR^!bE|Z@Os;#RY$oK~ zHX-5%ca3A)B#Ry~g^wzYLS8E!T(tsur*?Y1dC(ZEGSOm%)7B?klUycwa$OCyRctxr z`9s@N_3(*;Np8F*J#E3trh&#=FK`$&^{6cNu-babL&|mW%HCrKA{hCBd&sU-=S@+}RPjCHD>gCh64PxD?MR<>P1Pra zHcin{KJ6(T>3GYN{iRB5it0$ z!+Zb0{0D+RjQ6nU3+v8d-?P;E_!*t;(|vVyX9mTDSX53be$?`j_3^Q1&!6_4^prO( za>**XB-(d`W%kt^qqdi&QLfiK7CCVCXq5S^47WY`RnG8cV3gIQt+Ko~E*h*@d~b2m z^(o$!Hd(gm!TL{2l%6j-Z84Y8?>U3I;Y!v3gQAwGHvzWpT{2nvGp+Boe#v$1le&8O z_)9Lo?vr~2+jw`&-Vf-`)pcFfer5I1d79eMW@6XHAJ**tvdhVRVYGP>%bQCtCtgv0 z>RGzwtjYw_H^G=&iS=d-%D2#>n|DbbnU7a;;C= z4%fS;kCygs|28Wjv3AzNlNYZT+O{2=A&Rc3C1{wXO%E2RH3YV2A3{iE6+ z_Dr4aU#t=p&1_yT+s%|KDt4^u;LLepj~9N;d9w6^#}gg1O?oCqAFB2qKIan}BmXD* zX42<1(JT678gD(X_x#cRc$K@yy7(J64iz+4FJ8PtW5M>Bdi|R@IMbaA)*sz%bZO%L zg>vQ>PhBy%`_!|1OXZrrlZRRX<}V)k6Ty5+>=0QuW3|8{zm9=&CIPAVk6 z_wdtoiD5-EY!3H5E-KrnRL}jrOtQRV=Gz#*pb+t1(X1I;H#pC2;y?a`NycE|&Wg#A zB}I*|4#kKi9S&`h>DiScm&4)Nx-`clVZGS2W4E&3G@h?l49W>m(>%GDXV&Q!S!N^F zYd>8tGTrf6keIb`cgBQ<={{PltUTwweJ%=G*8SknkG4Z+BBi8+CK(iM?0;mkr#tfE zvZ@7MrIvU4q}YE=*nH~fw__z|-$vUV4z#OX*BxywF-+N-I-6RWz0=aaeGV88uABtj4G!$=gio5+SBaJ zjMul)4;5t}_L(luzxL3tlcu}Z9WcCZCu(f;ekt?bHA^>%O%e9tj55QnPkFR9S>xvJhJ`Bmy3XMh&yAmaIG3z((R#Y#?(J=gS6?S-9JFp< zSn)om?a`ji5>l_`o(x!1&gSgA=PJ+9isi;BpTi7A-p)N1FlGCsg&z5=&SCGT^0-!< zmfTcvTEeJ$xlzieX+|kurWuI5n|ml=#`YPC&6nes+=zajqP9aIP1$GtIiDFpQ)UMA z7_2oiUdZHlciP6YJJ)8-p0Tk}B42g(_51t(&HSpru0DJ3mYq-kO-qkz`E_ku*se;s zn>N$8oK>8?`rZ1fWv`B|+P-SZ*~{BsXsSGIK?TGr!x&Np=StBnf{w_4uLs8wJ2ULn_&B`A1fz_uBp z>FWv~%?>=W!#2@+>%xm`IUaH(a+zY8MAyN?!bsn8JN56tnJk`$`b<;fY&)8!W^(yOEu z!nkx#__y%s)l*hgu8VESS@kjE)WVu)Pih@H{x7?I>EB`h<0(Sn?>gr`{FKwXTJw8~ zB4gq1MF~Z|JG*q-)3w!9{M}-Fr4DW`Haa+EtM|c+hki23q-M?Dpl!Z=M=0-{Q(sM8 zcN;BudsF8`!wj?NX!idlWo0q{e)_Ziv;R@DY~$`*nk}^g%?YBH4|aV&dYL2M_`}0o z-(SM_WYd{A-t7Ofr+fPv$qxR;chldc|Nr=0;NIH42fexGEqTqqH@+9R(^bBrdc*%D z{s*yd3-9F}lD@)wVxwrK(-+%_cK%n(B~y6rWy`6_9}vkp{QI~(@}Ci}>UnZ|Gm+B0?Q517@$LuNEm*%iwmk4mnDNNnOM-KJ-9?_4@SeRq^;OI+vAbGx zU5d84Xk0vZ@W`dLqTZaE3tM*ZNb`uS2)iIKmnAsGF@0;+sa8#;Fq6YaH!Ar~Do)Y5 zk>YgJ%=6vX`K8Ndot;$_dG+BT(i%qC1|N@@l)SB;wy z=N@beuemjSTH2$zTc`b*Q8Ifa|LkXfYUU`-|9)+g`fLyH60Ub{fh?Y8q9P|)Sxsk3 zmL1fy@Gw*CurK&tzwe2~+=3f^&gT9*^<*?wnUdU`L9*&^zV+grW^38h54RO0t_rm_jmZ`bm|wfh?(#&FKdX~(omjQn zG@V`KaH+G8{F0?#A1+?^%^@IhreuwSUztkK=BtzDPL*D}adm=Oqpi10e8m4}r{n8> zADVr7y6xNF$I1_FZ@)76MgCIh`7<8fersKR?)xp-a(mZz%Elg=9~a$!XsL6r)pQn< z*o}8S+gb(1*Bq)ly5pcn?@`@PS~HDIznxGCRom3Kl6~$=_H>otiEFecd75}CGD%;W zGCA&lWxANOqM%uDcji>j8N%0O-4drA%za_RQGC|LV3Z`-j=HzkTeze(UWs?M*l1IxV%VPbZn(6h6tlum8k^%ol3idNLY0 ze?CXvKe6zV-Q4$L_aAbntGxVUCoAcm6Y+5I^f!l(e~pmj40wKKis~dS!G$N8t_U7yUmshg4dQ9zDAfRSv_;>QuBQHL` zt=+1Txy|zSp7#^BypQ3G7T>D#r{T4i!A;8$!CWO<6*T)G3`}T_?Ya!XIl294CAcV{Y-knzb@@l?$`r zc_*{I`qtdn&!)9mw%$I{zH)uP=aiX_XZSwP$eAkX?6A3@Nyp-gtJU6Z|9g*od!2IW z_4TT6raQp1kn-&(tE;iM}k8t|VPuTek91f$g&C zSLUwVqT1wg;%HG)UQ+R*2^K7EBF|eV&nsYhsCiduq2D16Ivx+@AXFAXFH1AK4^a%PAvf0AaGh431 z>&~-@QQjqX{CgY>jKXVYY0SU-Tvyxv4K-qN4u1M{e?5QC;=qnk=QY zfq@q<1_n+HyExIJIO|^D$=KP;?(Q$j{yx*Te!cYcUmvEp#vVRx9iAKe_V@MldaXCI z({``h_-5CqJDU1mZE8wxT`o+vdj9n6{0Y8Ethvnxo0@zmU@YDUSyJ41B#W;oci&M5vi*J$o4-l#M4lKYqV%YEtbEp}p;{kb=9JZ4uP zoo2f}FLGbn=X<>Tb0wUP{QB+gQ}cdH{@vV{rISxSzZSo{{LUGrJ;(oVwUBvVrqggdl^9){C~x#G8Nj-x!Dx- zPIF4NS52Xcq@1-^5t2jF{5{;)Gr?$<@FB==c`y7sBU=c zw1hK&v+1T(5Az*U$*_p+_TS8x&KKDHe)jrxdxE!XzU;S-ue?&|>)Kqh?tjcBmixEU zYwUmYJ>fq5d;6?qcI7$erCD|4eVq9J*j6Q&+NJo*Z4kcW7JdTMl83 z6Nx<{hn}3W@m|^bEut*qTNGcEU$n}Gh#Tj0)V3LS>vFzNJZjZ=)j4&Mg2bw)8(tVl zrKwzfGvU}5y)gGzt9pNVSPFc)^JBh|J;zO1{{^?27E~wP;c+pwFO}__aM1Jc zh2_)kY<|Ae{o37ty6^FQKmR$k30ghwKXftvXz#xltJzXIZ%`r(GzRyWsk%M8@Vt{l=5|?j=dx z5;=d1NjPk!r{xaaS;kTq=3d;z$ZmFX^RXvfi>2Z#-pT~>+gVlr|92_-U0vgq{v)rl zqLc&A*SV*KPI`atC|B~dIbVLPI~jgc@8YCK0b5ddR3>?z3kx?9tV-P!_;2l>9e?K3 zMEr}mcEM68@#tf=r+TT&m6mE%daZmEq*wM;t9{DiF2-jGNej8|Z$8w!S|w!K5n-v^ zXJ>zT|W6NRTCFeTh#gb@{z0QMPE+b zej@wgV`ET7pGwfqsvSmQn!zlIeV5jhXg95K(H5C}vih9OzCDWjv@-Bu}p?UK_H??8VwU#{=Fk+p_PL!=-~R%^O3G|5=;oy5K>g;5W^r zH#Mz`MH{O)YR~8O^6i-0Bj)9K&hY7`Vzz#m)h3=M=_@DY-mb$h>n)QpgmMc30PGWM6)hqUdFtYz2cHeqe0iXzLE!lPfnbCmab-NW0rl6 z>*eGA@As?UKXgyO;>mjL>$@8}*!9*3nAsORSzS?P_wyNd`w_nr4NMmAWdfxhcV%`) ziYhyKMde8?&Aa5fV!^DoD3hrxGb=NfX0F>ZZA)mT^ra^zWkR}1hn5}fSu113C^I#2 zv9|8x#hf<QT^?tTG{_H=@(=HshPlfV zHY43t7bkXzy60`0x!S-y%B}yxiA`*n2k$I9QVdulp4KVbfu znX(&aWR%IxzjDuX_q%s*=g;tQ3i`95C3pX?uk(yN_iSDH_T1UIzS|ece@)o?TkuyP z%a%72eJ`r%_*MP2w>n(W8!ae!Zjo}+R~Lh77mZ(BOP#K{We0Cz%BcSPd`@7Gb9Ssw zoNmy@TM`Q|zv|hfSEBUsjHp%5uS+}JZ=U^gW#ZX|#+>~Lie*njk8W3adw>3)8+VS! zFXLZ#%cAy7>*Px-|CJy2wf=YfpVc$}iYI$?_A*r&G@o7_tNi8H|F>I=ELn?pEL?Qk z=fIhxrpASj+3xa*mWdj;xA>YLjPf~VHMuP`N`Ap*c0O;B4gVRqIxMFeD6_0;F=VRR zeb?udad)?|aaVWIdzH&e!@Gg>h%I|LHS?B8?K(OAg-y$xR!lgWVdyI2kP^;eJ2~P>xY5LuXSta!D)=w9;^Cho z;UfEDi_|)Qwpu~YiqVNZv(f3)n*J+CbJU-#+u`I`QT?JM4L-k*DE zC3o?O@B6O(6RW%avEDazN(|%n%|X{(U(fiWy32A`TI_beiKZ84iS#a5DB9TlxZQ;% z>G4kErsJL>`)#wnd$FtDS!v{wSQ&98$dM5!+^!zLBvZ2LQ^20tx1-;f9G#avKly%|ea7?!O6H9!1cO5Tef7m`bvR7`FkxBUq zha6>z;uRWV2gOpixN%|*E%`r*RkI<#=oPTr?&7tz5Mv(mdlbG!UZ3j zY2PmH+IC9r2Ag$)zD&#I1)Q-TT^aH;AEev6=!$hjd$&9*Xe#$Pm3Q!6!9ty+yd#e| z6!%0)2kK_*tk3#7L;BpGc@vr5>E$LxEijS$Qp~C^zH?(mSDNXRqi3{|I}$`g`0jO^ zPVV^kIN0uf>*M;bhZTj&FPq&zDgADtbDhe}eaZW{_toA1og%_`|I*@mh1vHWPu-ut z&w}ZD-Vy7Ycl|EEv=9o7T=aW--?SIoUanYoy}m17rgQzAk2&5}n`d^2E>W5_Dd20p z)a!cR1^>?f@%(FlMf94G+w&7=jg}q_3lei)b2ZWMsu8ESNn%x`TnX=r-4((S2mQYs zNNu>wI>S==_y*o+=I}<1Z{7|M)B^U@|DUwm|JR=nwHIvzr=5%0RPcP?m2-u5yT7c< z?tH)G{cQP~_30eSzqTJqo20wme!lphqb*na9tJ$$qNz4p^J>Wn6*->&26Ni7w}{T_ z66?7T_uqVXvt!l4nFkhMv9P$G;c)3!Om_lsVhUvW`r$IJ`d8k1%JJ$Vx%l*1Xq*Uz_nWna*7wswYO zUKRB|j`lN0bVALJmma^kBco&8k)q7miS3E&ADaALYWUHI$Fo!Q&G9WF%ATQ|rTj0Y zoLDW>Iub;^boIQuQ>Q+i;JQgd|DVIbx`Zi=tJD7%F2C9n9=R}T>n(}IwPvfT7N2?l z<3)ew`lQOAS-PiRESw+iY#$}V@h3x2N3HRH#>BZ(jjd9r2>d#z6fo&yy!M9;vqCEB zo1cG_elB_|e%JET)@iQQfw|Y)vTyO*|DI7?etNxle565Ql<&7+@zX1xSsBkRR(o=P zW6o)3o#{TwQIldMR<53Vc`8RVi*K)5MAhtVnl2ZGE+m+qtY2{x)cqI0jaufJ12xv&5E&gEfzv6Z(=&vTUL+uB|I{_xtr_0_i~mmS%^ zq4i`v?>+nd0+$am$)3BBr+qQ;qPDh5l;*+g1AMM;4pzP2qVyp2feyb->s$uqpK}?s z&PCL13Aibg>$#C5LGWj6$dmGm91E>tRv+%(sy5?uvwNOK`kIXQuOII}vtgC4_<0le zzBcb%wa0cb1^H+G{e5-%vu(Y=<=^RlcPwd{XZ!!$di8B`b$$CJe&nyt{PJjj*Bpk0 z|10IWOLJH=_Lwr>r1*FOBK zw$T0jPj*h2#opbUGY|>g_r0>B!!6&MDi$#aKQ&sVVYYrke1(%3+ zsOz(2>K+Nsop7x)esZ#76u;!G3%l4fcrWd8tiEV$nX0^;xB1s$NnQ1VTSuNuxoj!h zxW!fOeCBI~e;)hpGsv2*uy5OM-Ftas&DsNV3Y#m0*e-?_{EPDIfAS@?Cyedn8lGAA z4OZM2n6kg1^7gB5*Pc!DGu>J^W7f?1_HJu4a&zMEZLA4g9<^i7-rUM+=k(W~QWszG z^q#h2&SvM_^70+G!*|VgJDXFk|0XM^uzZ!M@v6ILW4)zr>F={Ou(*BqUSsfznTt1V zVzbgtb5PZ3FN@&f{Zk(k-@CST!>pD@X|d$r-YwmajTZ=iG%DG~_9k1XPWz2eoIsq* z4RsrJ8_w1fx7&)hIBT0E_q|{d4O%GJrg*NU=1-sFDkgX7iRqRl3ObKsT5G17{mVHp zFKo%QgBmh3TVCrq))b!OyH})KrX1IKUhT=8lfT8B#LwMr3Q>>GWD8Ng+q2hu#~J(W z-~8{-Ih+2D!|Py*JZt2SznbdN5BJ|XfBvK?)A89m{^Yoy-tkB4vXxzW@_nNxC6A3w z&KvzP`w$soR~0y2xq|<)JF^q_x{XtAw}0KZ-MV6Rb8z0egRb_wzbWah?OMww6M6Qh z^v|xmeR0IVT<{-@S4D)b(_$j0e_S5xZx8UiawlJMTa04~}ZZ@74Cuf3*9} zk+RE4%mll_CW-s7oqcT=TG%E_e%_wF=olCEem zFS;rHd2rmD?U!^@@ixAS6o>b3 z9sibo$jIdDGv$fclYC)MaP#WzpEKS+v75mW9=GUX{H&=zowVn_oWAz9*c0C)Gb2k% zlg>wY9M3oZ-*?DT|NGIvn9tr5)dg5j7q*q<{S?bt&TlnYs=}o8miiCLr~96qDT*)B z`luoB_^8ju7#+dmkB#MbO7G}5{kSiH6W8v(c@)d?ba0eo1M}3I`$M z-NsRG-AX^-(a2Q%WuTmY(Mf-*fkp4;haBR2ypB#1X?arS`tXVK^2jGg-Wo;9NL}q; zqro6O zUok@)yuuwC+MvbN3g8vGkX5;eb-N12pykde%d{O~OX&x6iMF|^fdU8`nVaib7#qbZ z7#b=V8YvhX8|xVum|G|qnu3;NgXQ3Yu?mLfpp#^wdK8R|6^u+3Oij)73{637O9f*C z1(>93^v;r=r-Q!ukc zc;4JV0le%T?kYF~=4QA6oM8mD6tvdeKmo>sc_&uE!cf7&1Yw1RsX{b7en3m?F-Y%iYk>nAH9g_H=i_}4dQ4mA~vPd1C?%mKAs2iB-8JQcKATLliG}N;+G%NU<3d`YZfH+w5QVO0Ky(B1wO_0mTB1ms>YDojP1w-Eng~TliaRNKFG!BJ2 zva*B(?CDydq?J3l{O#l|OxK_9{9alA{{7#_e>_rExw_vK5f%kEh(XU~o7x__s}Gf4HbO>jEA?B4T8yN`!gGURTs-}!UHM`!iE zTlbPV8~z42>^Q1$%0fT{T9s{ z4m=Jj$1YVVv?=}SvY2*P^;MFzu&XuKT%P7Dc{3DvJFXKrqUw2 zp5s5bBl7jxPY9=OI9F6@yXL@`<%abV%WS*1usqH-w2^pi+TUur%5{Q*hSRYx$8MZ! zx>0la=Ijey{(Uu##hORW-aDK*YS%Tf%&p|GQK5&m!n29x33iq0pJbjXw?^LhZ!@v) zg~;OV$=1aN`vqPvO{@`X6E2i73KBllo#A5cWViL8+>%I(|BV+bmG3oPOghI~eCXrx zV_(*H^liNT^?==dhaVYrPi%tDmL~6(+??kWK8<;*$8p6;0()O3N}S)#>{z#U$~k6X zhm#^TR=qoV`P=77s9T)dcd0Bw^-+f$mzw*lAD1_@#Wr4>@x5V>%=5-u?$HYCoW%ah z%{o;#qw&*K&fD`lE5vT^S#aWT!Hf34=FYPYpVu}r5#K2AdsoPL^@Gwq=Z}=WZLYn_ z)qC()tzc|Htd`*Sj*kidBb%Nl-96b*U^mUSJ(Z0+VV5)Gx!4m+Rw%zpj{Y}8#_l_N z>+=m~l%BkfTq2Vwp1h)6DcvyP-t8NHOipep%Ci*DDbI86ao*y%r!AwUmigOhkxJK@ zf}2*JU_aINh3}KV@nt7+O8Cxey_UQ>bL+-!F7|Pi%01l&wQq_1Qc4j!zG>;c+=anT zX|Gl)-GAJ2XR`cJn^qswfGf8DWJ?5-14;x^SI^_xd)Sb>Hp1`s;hH6@1NzcAXNo>v zy}U`^>6*yA1%5{|4we?@u6w}|#P*>3Q_O*(J|rw0%CfN%5ELm%#VUGBKy)B;qTDV-*)zxgGQRT`*62!u;b8B%frm z+Pek+G*t4GR=V%yz~?3XLfPv_{EM>+zZX?DamCF4Ztwg5KBKK(uEXq|il6*nMyND> zu8eun^2+*>%)^z-XY3OV>^u>WK1uXd;gYDU@@I_R<#uq|aO5sgc=EA!hh5p0&nviR z+>8Ik^lg*t2Mv3%pFJT?st;ey#mmQy| z_f^a2yU$zS)N0YsV#fbf@UmbttN)a(32;KbW=Bp2tNtYW1p?twr@$EUniZHj6t__4jDRhPXwW;`N)Y1NT2$oI9)Ej9uiX zyvWK#=>ogU7uP9AsD8>|wpi-*LBl%F7qc)IL(QjSIABKQq z3V%f2%&+CnzA0-U|8&RRD#=fYd?rW#HZ6FwR3XY`;Su@H#6%IVS(aX%(`P> z=WAZ-ci8B$Tt2CWmFE#_M7`ae)9nXRwy~zZPrmSJ;*M(({!u+eliI%4A1X^$h(D-c z#!#BA*H+)3Bl2{L-8OeNt`D~tt_*r_rIEb$xNq{ZyA$%uAIy4G;rgTP?P+L`Crt#CY<^sxFLPo3C$LJufuy^D&1&^aQ1g_zsCGd{q=F+e9f|vlQ<!d#kH*VSy;xLb~IV8C7 z?OhU;_Q$A`syI{U;e?B(ksdtIr%^So9z29 z`{36(hbfwW{xZHXlr)<3ctTeR!zIQXE+6p&6~R5zuYI2U;AQwhs|qL2{D}P-R;Re+ zG#_r8eL27H{+@OJ_58LU?7qG@j!){z!@?Q52e>!dJbJd{m%l>;GT3sQ-CC zK_F#A=&2bF6F1I&RHT2$SkQ#`_^cIIA~h}u*f;I!5lEf-i!DNB!3({nymmt--k-V` z>y+0$&elE{{wDb3`uO?3N_SS)7SuTYV@wiwCo1nyyrg#Gw)M+-Kgn(uH$PGHe#IX1 zBdxlY^}#dkOCIsxwNUyt%Rc|}f7J!=WSsUiSN#6}eqYN}L9;_TZ;ohwapt<%Z`1bg zbZuP}h{Xqq<#MnMYzML*Vp~~;jj5i=WxNCyu$O` zFP+WjEB;H4+CT4q+r64=E8_pxyRYA!ULf52iSfKo{G$8fyZ*iTe1C4?E6sK3KdL_b zn`re#^Xs90v3+Yk%oqJ{=r;B9>_^9w_A_j&(yB>+xS#8OObOt8vm)>*Sr6m{p0&yyO(TVQtuU>vUiTk?x$u& z@|AOJ;?_-&pZNTg_7m|Z;ZM$f`u)l9=l3a=n)|DL>LQz58`gPc z^Ze#<&Xb%cZlhmqR;8Lh$Nb*xSG*OoD}PtM|D^YMo%hm}MXNNg&dT!l^(kF$=6BX- zuWf0@D-QXuGi&4Qf2@puZ~i;CZu$ME{-4c1KL5r2=lc)sznlMl{>%N>|Ht)T=D)W8 zD*yfc2mg=zpXEQ+Kl^VLAi%+9#d3=IFq1O#V?7V?IpVismQ9boazV=9clE5MS!Jeg zv#qXvS*5wQaz~ou*-f)N-rg;3EwyjWR^B?XV+S)|w#d1)inC897V%nsH!FG<%*H!s zcJG|%GxMIeNLS|9iN~Ls{yFrIqV6oSI@P zw^#k#;|~=RiX~6%%<-@5DNdePd|vtawfhHl?z?Bb*IU{m-SM32l4VCM!sl7VJe>1< z|AUx9=I7IXUH@_2yx^Tc(YxR$F|*mu^>Xy=br3l>@${ZHh37_DQOeJoOl7mJuJf#@ zT(E3aTA^?5+t|{#6N;xc6;E6rU@+Y^=ZipXbE>P~#qWX2J{lAICugR}%naGh_Bgra zOq=mx$%n@tx=GY<$Th4i;O}XBcVXSP&$oiNx@X^CZ5#bN@vUIsrn1zxUAKC-W`F$c z^0w#J&8_t-WrJrke+ZR4Fn#%=@~`L5IqLj=bZ+zJ^Z))G-L?6$!8a+Ux36xl+RR^X z`1H&TPY>&<_@e7byoZX83wFV(>BJ&#L{W^OsNeD-|4dFL$WmaY1{ za#1kr@=yL-t)>>Q`nmGh@*scJxtr!Hsn=WlynB*c{rvntL07DPeml8W{ePKIwdK$6 zllAA_soC@*rG4^S|7CN1Y+u^WoU}v#W7H$@XZsf}|89QOdsTXf|LS=`YlGH?tPPM2 z$qiZ?y7t)L*U>xYwk!`e^Le3{S^Bt`>sP6W`4RoiKYm$A^Zh^D{?*0%^K;8?!SjpU zuT1{_@Ttf9eRk9Dw@OGJt^fM)DAOK)ook=>OCR?&W!@9$Z1Hx#?#jRV#k=kcq<*kD z(`NZbFM==ra7+Xb_XEia);I_8g`8ka9WVDfQ)xldlMr9JRmeDhWVX1PYM z8Jz!H-Y1Gj`u+3LNNf@2%vvyQje_<`txqCX9A}(xalIm-=s!9A#O4&!TZ^{3IB#)r z|MJB0lTA!({7LR7uAhV|XRc9RKjCW%^Gy%4iQyO1T2F{=nwrym|AE9xCGHbto0{LK zte=>bq~Nxx`;D4flA2gj>*f!voyumPtZq8LQJJTsl0RYp#Jwr%#V~akN7h^ z;jr}j=V+%QKe7L!vxj=GNAe`b=MQWw*^WQtsbN!ppfab;|3Pt4XN;QtqqwiaKWD}$ zhM!#j#Qc->PB$Y@-j%AQDn7Fw*s)Cs@REEQvdAbj(DZ76Zl+ghh(GTY58o?3x>vj& zujn;dTqfne$)oty0>4=f!kI#)7gJ_A-M!*09;EN9s_S9)b)w8Hx4$M1mot@b%}Bf8 zV>RDSYo;pq-Xm$7B6Ij=AD2t*z9ICDCHh#{ru2g6H52z9GTY#rV|@Ei*@oCRyx-J* zTU(#tTfFXHvuyJGjpuKQzp?o}EjDHPjs7q0zl8tJj9a+>!s;*Kzoh@pjazKGWm@LU zzL$mNiWV!ye{}y*{bv+^JpSS9pKSXk<{yepHRk`QTciE%%a zwtsm4neUta{&D`NzkhV=8vf7SfA;>v_n(^o2-NrQKhOV}|A%1Rl>g23C+r`|e@^~m z@t>u9<@q0+b(7{ln*aIoA0GSD`ya%ATK~hcZeIP_{SU=|Z2vR=kM%#>{U_=lO#hMk zPx*i6|1;}9#GeR%BD4Qg{nPy)@BcK7V~$SHUaPco19RD-=nAV_Y_}gW{9(Pp?`xgHGp}c`T?2zk~GvW^mgE+iBs*R@FwEaFRoFWx6 zx&4IeC+77JCjaoUn>lrp;~PJ-8B;&rtl^q}?D>b4HG2E!pMU)M=i)tG`VY8&mi$wR zKO+8Fb%(P3vNtFFKQ6zxyi&||wrGuN{fza`vVTO~SvfylH&XkXv2ePJ
Wh6^he zn$MIylM*v2G^;mkxw(XITDR%BwEl7nVWG(}CpGi>&96>>#=pcoYtF=*OENdFd8WIm zxF~wtsqDG;m*pPJeaqGS*1F_P$d#R^-^RXwac!^r`M9Hde}8#_*D@%#4sJ=GsBRKJ)1_uxd}kK^s~wKZ>ksL!t} zd-P=GdH?x4pIYDhB%A-UZ+p%0=zWiM?|+>3zT$NGo~PFT()T5AsQsOgxcfHOt!*ds z=Uz85SKsaWc}<${?36T2tf7@;I*Z&&$gK5w=MpZIvd^0|Cq z&YfEuQ|^7;>A7ar3Cm;3i$X&`+blHQ>%H#$*RJeL&25dPEB6;yhNi!%QZ(8a9nb&G z?fH~xEhl8&TvAR`N)w%XpCf*vho<~%&deIwn#HS+O?!MvC9HM9`M>wpc6J_&ow#{I z#FLe;!XJHZeG}7 zK`|}6zGgnq5mf$E>e|Rr*;ZMgqG~rMIJ;2tqSUIkt;dpnXsSkNo5+Nx_;_62xaZUz zwx{^PEq{5H*Vtr?p4hN=59_?_j)fX?n|vCwt#?G}$e^^3_0aq5umh}owsdy3o=xYg&WBnB$`+uwZ-el_r%o5>JI&Ia(q)~ry zNyL}7CeMupnsqHDX1SLnd1FIE)8{WQ4z0X;;g!eJyLSCI!?$+7TmE~sn%LCq+wCQO9wO+X*kriAIcu%p1iEBTsS~ z9lv(Gxaq2gm`2L&8`YbpzPON4QZQ5KuGt0?uj(&-m;7QEAHC($z4b(|n)z#ir7wLK zU3qnWok&RbiF?=W*rvSC%K}~0N>FQ3{Tc!WF;T?nW zi)p)*e@ooV+K{sS`X!FNEMWzg<4cto|u=lrAZJazt;FGsv8KH(&y)TR)o{7p$uG2~0}o>?z77y6$H zZQ&N2GVjppne&ch&ONfabKZdqoK;Ni-!5L;^XE;hmx8gjSi$dPM)x3{gx>S(mdWS+ z-Mex3th%jdC%=rlywCElgcC>C+}fVrdLB6!!$mTNf{Mv6EZX{yq=Aw`(=?GS(liIbfBxByCCL4E>Ad+m}Y(o%l^`m-x-c$Fjv130&khWZB4~ zczLf$>Fa{+qSBSJd@C8f^IG#oe+k(vZ+j3?Zg4>`^^CG!quKUE4*k`(tC>wCBK8R_ zek2s}$D-|sszJ44+Y!|d2Yh%sy&ue5DIqldz=w?>a)F|R&~%6ObrOfXO>4hx-BMB> zx4!Vm<$Es66{jzQAjajFkBHYjPs;JnJ$u5TZ!K4v*|~!7cWP6sJyPX^e#)$ranzDm zoNDZu%D8e*cSzbKFY%Q=ic=kfep+Sru34dS@U6Ma=Vrg4%8rmcrKyuWQyW5ld8SIO z{4!xrg6Y@1x3d;cP-@M|*8jrwGWYE!Ukf+=qAT4=Z(XOIM<=m*47sfVCRJuf8aB==abJ6{AuXxpF$3FA< z{5Gevdq$AQ`Y9eF7B4)Gp?Vqb3Zm(y%u;u!Nqa~awkDYXSFRDh@`lktkJi?2+q;SjLDbj={Lb34oQs0%R#7kNE4Z{Nk1V0!tQgYfsfB$2IU zyXwp57T3RCwM=5qgQ~lKJD1Ozy!Ku4j7^EledvZ^&`GD3ZCiCVYjM>2YpbqiMV`GH!kZOc`Xt97_zuIilWC4eteIJM2$=;Jihl2E zjaAO#DtFN6D_wl|SDC_Hr{8Uk+jrY8TwPmx)umf{#go?#PS^4-tc`CEczq`C>MD=< z0dZ;DC#?E$Zjx?`j^5EL0(0Yi3%;*b6P#$G?zv;t?goL=b6h=}%w}y~u|;Ig$6)Tc zt=Gz?{W&-H-;*G@-D&?m7Ty!^AxoXet$2X1En3OqiOu28H zvt@y^dy7}CvQ=x_PpispTV(&J-)wd*i;iEeYp`QQ{K;tt4+_rRwvBh2?kC>2A-mRk z7v|mLt+l;%Aw2Kne3LL8snephr6+$+eq%H7uIrbn>c372*OqdY1^!}kJ=X2LZN)-o z&0TZ3?z)vmO}YL_B|o#h;`5QRt`FP3TFHCfI(xLc*6PcluN8}o6qWM5a>aymqe4{#pm-{rE9$w6T_T>Mo z=1SL!yw@f_?HckH_Uyas)cWFiUf@cDtua2DSC-lNF4uM{H=cEaNBDM!%P}dgt}RRx zmt8!<_3!mB|IUkjPTX2AU*GmHXuZQe**j8WX|#RUvIX&HR^RJ=H|d6)+Vn>(>XUtp zcJAX%y&=HcJR{%s-E1N5qdyAbd)%@Ur=I-!CgVqftV!slJP!vIFMVew{sv)%PrQ$s z7CbbZ$gLC~@XKE(Mwa=2)eVJi!=Qgi?_1syzLE(Xhd)$Xd}{uZ`9#FzsMP5M)j5j& z*<}s_)hmxoI)6YV;^--h&YrHA!-5rwQhkli{1!gnF8buuveY=wd7^ktFuKxCs7vNR zke){K#SL6#lcz1={_54NY^&6+DEE6t-`aI8Yc~k*zxsh~jzZPTq9?5JT`gg8#S0M6MnHdM`*>y<6=K#`I0Bx*WU2; zvWBVk%6W2$=RfoAar5Jw-z!n*%Jf0RhB;q|`P>y}waNP@Jbt2mYa-u5|BL!JC;t)s zw=hfN+%G)|{_X_kKVEB=n7;IVr?U8@bBbJL&l;8XlYvEz+m`=*VNuoDnJXIDcyZ!0f zz3=bI<{z~!I!fm?zc1LHtnYqu`jhB8jdhBqBIaRQmsJXq5{_^q{j=KI! z?_a(@b1*1UE&C)_gy!sq#j}o2dsNQ;yGi|%vQ1OXxoOk9S1&ES#Fn|gV&1Y}+<7nG zU$={&9$0S0^YG`xrw`XYd^=J9KwoypY*x92Tm@g8KU#}jueS8MllG(I#l-SXmF6wV zcPAPYwDZzlWb2c^*^C1v$=8C6hrgHx|!^^xc8S|6x_m8yZ-pg zvZ^_8$y&>j)zZ=>*KlhmE8nPnQ@3qS>y6)UiodDuK9%)R`eD?&XDe+E7*~qhwfH|g z|0(~6`K5%HwlDXrnR@(m@zboI?j=dLR(6H%4NYC8x@zvK{;N+P>1n-bxW2CbisQLW zGuL#yZ8&PuuKu9&Df5))iUxDOF3?z6B0JMxUhQ_~ER)zg@7t5+6-f4f^S`wHWmeVH zZK<{%=9k2eb?pxkmA`rHXndXchwTrP?l%|zW{94r_xhyrmq}~d*Sh@No%yN!+MM)F z-yg78JBQTh&Uq5`NJ0GN=`|8dAMoDu(_+-GmWoRieUf;?v72Fw>8n8&i*jeW zZCZ5KHGkf~a&4mJL-s_ac{29Sci-uiE^}XUZsM{nU3G51FFiG1r(bm2f6fR0Kl`}m zE}Z+tD&(X80(}uTF}J>jCFV}tyM^Al{r@6!?i2f)hh^LNiw@booaT{kCiuLXt;#*F zZ%3z?oBO-QxDD)aA)W>yQzc(NnD?J?-|>$uYX_+|jg!4W5&*Zmw0$1K4t|?m? zuRpvZACSCn8MBv<-^7n zAhL4@(*yw~4;#h@RxQu3zP$eMv4)Lrl19Ihspx_BWUp$D1O27U*BLHtoO{gTc#n`v zu88j;2H*M28D%00uVoyHeNG=MD|*Lp;wsY?r*IX0jtjf29qjtHGCaS=_gr8O?}dfJ z4#)WdOJ0?5|6d+ZqVMoLG_WMIoO^o#_y6mLU&ZnsVxoE;uPN+$lPXrnv&wD@F7gWt zy*%}x#e^T0Y*C{2?h1~&-p@5|J4ZLPtx-99U}omNsDSVbmur?TaF1maNxBi!JzebV zUa|E@`^A}LvTSqWa|8CIynp!n$mW0ThxTUf{31Bx;kE^4cU=zTgswdMENjwc-Z1H_ z9?fD`vmVDDICJK(#O;jAw1qELt$Fv#_4C#48~%!Aj;A~tqKZC8J$-sY>(8yTEpsBj zIrckNr`?-hH(P&lN?Y}N#r=Yh`PE-#^-T-Yt^eD*!TQb4mG!mCw@&z&Mj11%cI$E2 zJxkD#Nq9|K`OKa?Wz)PZ|0@3%+HH8UjQzuG;rAWeXJ_8^nKW}|RrD`rbqzkhvr`AHoMo~Rc)ox9)MQwX2@D4}5klUL_7k%nnT zJq>rFt}YYjY2-9uzV_hKnu}{=8yD{gDtkKTXHui&P2F|Rx}WXpK9Y1c|Dw(E)V{lF zt1m@wJ$1WsM@2$eop#xef}>3i{Vfa2llH&$G?n!}{+P?0Q_W|Iu;g3Es)Xrw3py{k z=u9}Eu2+=y&1Al|aXoLOt=_f_K2eL4)8eYDZ_NmKe?#$Fo_XA}hc6_5H2Z0>MGMG2 zN_&uPP|q9gR=ITz&(9J=s}*{#;`3WVPDZjnb_vapv`ACwV!1_5|xs6 z%ac!Wy$+VAitL|~JgrQ5Id6gLjZ;yJt&@WBD%j;j1rW(Rq zHFKrP+P;+z>&+$j4D*tEjF|)WHm<($`b%lR;-)p6Sx5N~z5H6>aeDR36W3KdrG3ht zXRY>(yRd+3*{`C|srRO>TBcEt5O-f-l)Laz`p7-@o5RDZXjXrk-K? zd_8TUVe1O3aBiKSKa)5=KQ}%7#6o`Z$(GMOKP|7E-_s);m?9?7YvUW z9yC0BQ1+@xZuB#2Zjr=Y9aZ6B6DvP<39LHQU7GrQLTqHJM*I@fI*v_ED*^;-S87Fu zZOxiGIoCCGLhZ@~!{>=Q^KLwPT)ZiI>zSiUMyqc=%eho~r{UY*_siaX@3|Fc{`SQi z{X@^^Rem|N?ELFbRl4siU$#D%t4XkZpDlBHkNf8``#Dl!a{s;_Ue(<{A@7Oer&F_E zf4g26w{72+UpGZ(n{!* zi4A7{w&E^I_|&cEx6bsE$5YcY<^L} zKH-Pn_oj7k|G~!kb!rKxd#3V=+%Kh89(&l_*mI{grI>57ak^)<^L}D8`N3+WvdUIy z;<6QNe1|sH#$FTtK2uIyQBnJ&Qr(=+je(XE@^^e%vh>IC+t;?GEHOKDZ?W-V=j{`n zf;Q;d2whhXY8TdbwO+ua?8(V@+Q{6zKdArOl-bWWFMZa0>GmhP aJbf->Y+;YPv zO5EuGn*TRHYz{p9_0EA_pXFCQ=Dg;0+qx#`rQw=sU$soq<)+?id`OS~cFD(<661E(Q8GqknSk!` zJ!yW9A}gI18=bFt(D3m zjg3Der`60c2t49Aqe-UQ+}bXVW5o`smHic)KSyys<^OR(CGGjv#TP#mNkx3W_hjPY zXF8G*Kc5}?ApB(8%(d@dT>GhizRiEq$8uIHzO2jnHxGSK%PfDnPOyCMl*#*+7hRe5 zNo-BOYF$eII$^2nYXVB{lue zg3`uP(df*Wr%~>43tsCS?WXww_n}Q^&TQJmb!D;V5yj&8EUBX7 zv!+cxE4!+}VEUE3&0LR9ZhyaD_s_MT-4Ffe-SV+|XYV>|%iYM4_tKJw zy3a&}R}?OgTI5-xK0EpKyidn>b@MJryclirzM_7EsCUrH)U8XNW$CF~2+s|@Hj^jMIPAT~ZI8uOJ^jw>QF|tI`X#2Z)ujo!ddR%k_R!5S(B;p)H(`O2 z2V;-?*7;d`c!9gu``LFsHy-CZ&zZ0Ivw!c6MJtZSzkj(?^UuC5x98pY`|iU;X8rcV z>v_)JyRypl;&%Pozu9hG6Plg}h##>|u-ajqF@34X)~KaS$Bw#eoh2f>A~fl2Oj`E3 zRr0$JFa9dA!tCM!(~Xg@Z{L%Q)p!`7?Opmb=Gj@M{hCgr0i_GoTuo8Yj^o;TO7Pj#QL@D zdnz#Slg=7z+5IXtC1;mEyn1{4#7C1?b5`1HxLN)E9B1)2|IgdzxxYy6kI8fAHlMU> z^JKO6O69dxbA0aEt@?HHnXq>Cmw9)y|Gi>tO1%x#z3MgP)s+=Ia#OwZR+|VgUv(7^>6j&ZdQnUz&ynVJkvkI?HE!1}O`8_F zKzWr$_|27~B2-7^;-3w-t2J+sJbij}aa)ET20#kQA% zGB>wuYN&oKWb?VDA>C1+>mg4J+sRi4k1V}$xA;%R$|R`^zwbtMT4zUQ9$nJ3Ytye% zp}egES7c7#`J)8L$=k@&J7vY+L`{_v^1=>7XJVGrXk58f5Z$1gHlulpf> z*Qx4Ec1xvGPyOwrD_vJ^SU!I|_g+o1@cN=_H)l@IKXWd)$$C~&)*GA2kzI0JAyYJu z%|2os^;A&bOXH*9X|73b?{ehkH?T`9?Uj?;U>T$IcHPgH8*g{dPuhCj>ht%ks85?s zxV0^KRp3g?&kOWppIsE}Ul$R+ zZuW}RS3IpurF>j>eu*$nvs|n+b7e-z(Wz`xH+3zRtxDPxImOXs&$&IZvTx>`-_SUB z?&qrq-((nA$4{NLK&GQyEy2KAN@V4}7lx6~Plrq9c324R-(<|!eDY#~5!+j(AGgJ- zd%_RZXy)5$_c#lxInC{S`m$W&?q9!LiJQig&pTWTxfb&1pAC=h_ve?6SDf-_l05jy zMR?=P+-)lEK|;^d@-ADJc>WPR?)Uh%+p_|T$t%YW741mq=9~Xuj>2lxpN@#>%o@DCWAga_S-Jz&ATT+`Yi<7_M@T z=U$GUS=!Na&t@vBOY7_unJXya_H1(4`-$hIx3!nQvlY6x_g(CyKPNpWmHgk{Un}sh zMNaAQN#oy#_2*d$$8CNbYxE-jVfFY0p16U6}?0GlwT7%Wa$sQl#7hM+EEXaA?dWmRi_|Xrq&C`!o9J2{< z=d$v<@_K8+j)UtL^vb_4JpXv9eG7lv^U3VrZKAL3J^JRu;!AmpOOKrUxW4`MD>I`_ z1;-vlmNEBjQP~vni}CsqzTOVgA3r7>c*Wpb@a*Qwb4Twc&EC8-*Ur*8>(vRKEmOH< za+&Tfeb=|H(P-uV^(qFARTFF#bW^yOrf%rk%OHKj&_b+U?ws{GYvI@+wG@wB)i^cV zcQJAnVh{fP=yAxasavpm^Xk`K2XZeYXIQyOWy*_J7;Lz) z)uP~1rPhQ5_rG^$P2qp^?ntiZ&nZXElUBJgrrXFheB_v@5h%#0sjGVQ$O^5t&pXccbnj8miqq>T7RE-P~A@XaQS}8x{pWtKFoBDO!c4qXRXU`mmiZ2#Q5qb z+%sbgPuIOAXd|^iWa*@Lj*~;vRj!4tb9^`P)yKTNwS6@^&(50B{pHiPD_>%|JJgpy zpOt(?WbusuFSk8$Fp)n!J2~VDyL$40{5^+D+n2A;+mO8{ds5o2k9C)2-qu`>=UqQz zm9Kcu{TXN6_TEc2ys0BL`RJ1C5tCP#8^&oz>!vlw6=j)x>^ssT($2hRWtWTfT(!x8 zbA=4-bf;HdQd<_fY~z-Ybe~PNx!3wfFJH~G@4=4CaLclI8==85R8;AMzwCUqR~85Ue>YM4_wW1G8&+<=EuQjr&Qsg} zziM{u+_L-n?(g=sHaF*AEB4yav}MNL4O{&}&0C*7xqCnLOUy=gp1zvH`Wxy#dXrpGj)nIV${ zF5EX!Rh{Rl>1*oQc*NR-F~dhnJ*}EKOlZTmV4bXPHus-*TtD}!+Z4y%v^*kOxS8d; z&m9Scjlb1ISGW{bYMQ89m%a1!=4Xj|*0nqReWr&hR%Ey`YJ9sX@=m9ZmbirK@f7-t647 zc9-#fXWOe^CY2mqa*26s$mQ$JdXdT87mWOEJ{}Q2e)-dn&W}?+>fBo}C(mDe;0jux1@FYdwIEaKYuiT*xLDfcTUxh zAAXtTZ>OC_|WZ|A<4 z|0{k4o(=kI_cwL#xyAa|g8Pm$+tg;+WSwk#&hll3ZOEoTtxS%o`@UveoG!CaO?gFB zPuz8l{Sm%eSG})FsH!`A*_ZG`v{oYG!FQs}q$rt@Ii~YH4 z1%G7Wx;09{2QvHat?>8R@uKJ6n*HziLa+F5S*X85)aS%D`R!S2Tm(v%u2#0+l6TkY znUUF{q_rZ`1&_QJ*N9_!w;^tl@K=Unj@HU|hZ= zZ2673X1niZUd=YqEYoINE4KIK6T|47w9Q9XU0gl4>|~1J;uz1ey+`JxP4-+IxLWg8 z=JKmq8yl=HKJ+lzetp8d2aL9HZ#xt`mVEc?-?4Y{gaBW$oX(qXj?zF2=>+8n2%>LJVze{sQJ*ss3+{j#^X%Qg5>t9oL@}aF8J5;x1 z&AmEt^+ndt$w|lLxy7snB@9z%+i9>yYiC<{>LkRJFg{-6@u=uRulRNu&PuaC53JKK z2}NgjKAtcC?bFU#b$`Qt+`E4G{nFJk+depkbROET_gk-b?mBOt`v0Gs-o6PfNjfc7 z~DSqraK9M8@AQq3iCe1hMcIkOfj z&&)b|{pIAn+XX$oiM`mO1C6&^OO2pp?q%}!);$Cy?uCceyy8W>v5gQm!~s!)(FZqOTR2XpkTN{z4g&gLzNGG zZ>P2&|9ZYpGH+MjKd*{2@&AgN4h@UQ&v`^5@=` ztg7@2bL3hYEFW;1@aOP(G>ZM~y0i30(d_5%PcP5@2}V;&ye_2Tg9u5Qtbv#0;8w>c@a-Tcp?%RzrO*gU;$ z6mD~UO6beCD;~`_di(9Rigx`2DN_agXB;X#;It;;$GVNPzn9FcsXkq(?ly7GqMyd^ z{P`^3+x~j-=j+k5UggsM(}mv~Ih!{h@0a`=qfi<4|D5!*FLQ2QPBT$i_H*lZv%PD# z9E#kyAiCg%?Ac}U>t+^5XMd_xp1w1%f6eXF@86W(Hiqn2&m-D8!0D^S+(bNco02B&sMncSsO2AjRB&@ZWxv!Vw-_~*ZobT9sGZ&v!aJ9t^%l(Or8H!Zn7$$9eO>4%QbS3LdwTYSNf zy^HhZ-FS8SViLpyUkN`f*wF3fe`j4^p{#7}<6l;%9-or+zqtH{)?w~v;zD1uoSVvY zWDg(n3jQL@JGE{47i$yit=3aJj>9O(|zKy(;*@HK zjhdp5UjABg<9yg#C)VS+zaK5p7FWM|KYja?^qC(e0?XF@yWt>nvHkVVu)>H83vG_H zjZL~Ir^nd}{wuw-y+6Ag7y$rt{7)juP*d&Zf`^gWzhkKM9KjxLEEfUzB*XhI-dNyO$!pIuwTbC~Mib-1C-nEM@i(SG^ z^4#hBN8**kd(x89-R9nGnY1)GlEJn6io%3Ham;B@V^Eqhlf zuQFcM(Zzl5Ymi~6lUH7$&xWcPmx)35j%Zpe|M7#VzGGTIR=)SbD=V%Bwk@0?c)L_z zWs6*4!m*8-$@cR?_t|7e&rN?kWu^YQ>VjXlcAnQ;Q(0%47`pd%@Z+uLuKfD9(EPsL z$A64ZBF(QH6Pco8{pn3f!4Bz|vRm_~dZy0$w@WZ*j)|kO({i)snKLqXeKmeL>uuJe zRjpfnow&oRv-^*HTX*l;LC)G3=e3W8+jlKHefe$p%h2VLk2+62C=fZ`?abRoZo~JTo~aN|fW^=(>> z*{EoJa-@&AarKqCuQS)3II|~e;>tPQug+;UwcU<(b$m0oV@7j;;mWuhi?%c7YOODd zP>Oq6%Rlq(*4-!hckknkSK1bQykp{(xC?o6jdK$3a^8x*Uv*$H|6V6Hi+T6`?qqO( zSrYi8VyCcZy7&|I|zF4;pe0ip*}CA+$+omO6{Dnwrhh`dc5m z9Oq_aq;ROd@qWOWRMsnWcjw70=F78VqLP-++-7ku<-)pcg3&5^CuHmRdhGnB{62g; zYRkFMcf#4R@9r$UasBhT7TXl_iicVknRbPRpEff7c%|#Np4?QuX`Ra1jW_D`7nQHG zIlF15?j|?Y)rH=M*=sx#Ci;4^#7^p&Y3!-SbxmcX^Ybld&I&ItyW+gCt?c8z^VhDf zek_#k`_|54Wog-bKli>3q1@WXo|egF)Vkf?`R85hR^`AptCs4`^^16xII4Z{MYNWH=UgIq~(d)u7w#Nx34I)&)0&jm{%}J7O}Oft=EVw2UO!BzKKaxrvpXoZDtC|V@%f9tTF-TtJ@SUb>XhtM zzIQv$yZo8(%I|izRgzV5*Nh#vLo!~qh8ZeuK3N;LaKekVT&;VQ@@hglW;2E#?Jjyg zrM!(rx8rWwx`K}O{ip6}<|$rUk!E$_ap1cWmPhM?&rh6PdQ~s9{y)4iCB3_xJ6IFT6UB z<$r(M_y5}M{D-g3Gbe~o`OO<$=D4TA^US*wUq4S<)A81nz1lPu)yX=C@^3X_;%t>|9&q{Bio@mq&h>eq8v&!=`_^S^b-d@?!gR z?rs09kba)2ecm^{qk-P?)piprUHAC1b}e;sRz26+wL>y>@u~bNUpjhKmVr&ZpTziIyUYToVTJ=$?04Ywco_fOyZq2#W>uCja0wc_?M z->pvFs0rD~^i*>pmmJ@;ua|VLuDRG^^4lccWV)%=g&n5Crc-BK-MD1)tPZ!=*B-g>#peE+yP=H7ArVy-)f`;&S;&wKSt(oZYk-s;0MC;pne zF5=UTYHy3~pSQQ?Kff<)8I=6IahC8QrUw!?^B?`xs8$pje=m8Rz5S{E&yE_ue_>x&|Ji*j?`oFn zF^j3GQ{5IH>u_89ZFS0QFID4neGm_uB^FyTVnofnK5ORo6wia zDYkZnnzPd9%CC^GRd$HqE56RN>E4Id?Wa%7*t8&g$sLn5EB0qgSYLZ6_UwdsdF~PI zPLH`W-FQ}G>HHv+3^A%L{Mo{1y^E zb8B&((!cwRugWcidjBmIOIoRxYCY%QQco35=YuQ@EbZ(qxmn%lP7|S&RE$&V2b&@~q;?py+G5k&{Xz@u*f2A(+|6b)WRKR4onHr zoNM<~h*W=Lr+>dG{L=5$yV6bUCcW9WecJBl?_aIj|H}UL|J|$7kKJ6%z%lpb9#*&G zAt#@nzTf|Fg3UXx{wG?8y7nr(4tjCMZ>4qu|Loc?OMZ)*H9oDL;XZr*4WmyEFMHSj z|4=VGYvaeRh5M~)V((uyFQ1{ke|u5z`_l7ot*zo*<+^fT{kV7UBm0SiZ*Pa!{diK7A|K!`kAAf$|Z|V0vLAPbbMy`q^ zcg8Q51NskjGG=y`YVNf#Mi?*gm#to~c* z9rI%HW3M~6pUWR*ZMvFtKjGE3l5XiGGdwqEB>4!{9!)#Ps`XqmefOL@Pes@I%DBxr zvr^#S*~2oFzR|az&(2@x{(kT0tMwbokH6pl_2ZG5>HP2Z{e7_TcA30NUQFrv z6Gzq`zyIs6?hcmQ_iG-XjonZ#F(8A{=Rxi z$l8f%KG}L1^U9uXQT%E*_5FuZk=UoZS7pfVcf9rDcEpy}`&zC~cjRYZX3@pI4lIeb(>4TR3k%D{?6D6^>dhE)?~IW6FGy zjXG^d&OXw+sFSu~ck31nQI1%nrMj(C#NxGssJ&&JEd)QyTX8-nsUyq#lCT~B}Q+`HARNRi&v$EgKyYu#ZTY@7 z{9eWT-s|_->vZRK%<^O0pAypZ&u+_cw^tr({~K+Z)8l!qlJzswY3A=!ZGw_AOSg-i z)sB1-7s0}rA9zGo^H$8J)lXtySD(sXGv9mv=hLQ3d{$|5Oft)qjMQ1H9{8#8i}ru@ zd*_+=YyR?obR=!}a)Bb__N&aA>_tChbUV`0{B*3H&z(IN>$lnX@h+D%v$aq5LyWj-(hj*ZH4(2ygME{-RFtD_Ttzk$@2!=XNk|) zedh18o|{@bDjLP# zLibLtw0I?lPv^_6Z^X@+CvsPF@#*;UycQosx7(JvmfUN5y?)21Q>WK`pE5&R{_eId zk58^r{cN`U*!`uC=d<7Z!6;EL{7+%a;h1NduLaf#iA`Itd-+ru*LRNX)9fGqEBx!N zw)=thm*2CdUH?D%e2~jE0VVU~i~A?u*T@kuzh&kn{WkiC?jf-JpblfcUbMi{=9_5lNa_hpR+E1bLU>z z$Gg|%-psLI{>?h}ensNm?EO>v-rafh)lBT}uSd+;i`Umae#`&i>Cx!mTZc}k|InT_ z{qCNklM$tvcQ45@|GO$3zWuz$yk}ykqo2LMv*X{x?0P5DwKbpad#ruxcw$evZMVxV z%MFv*E~`6zX1pQVpyRNq`$o&2O##nE4u;Ok@!QEWHF0;4;zQ5Acom&nXLt*D@Az$D z++_JoFwefc$@*mE&gUn@o*duI|K;_uLvb^k&OA5y%9Q@%aqcVD7+u{Qe@>N$qIQ3O z@|;~3`}R$*fYnp z-`^@>S97SnR-1d~&C3FdKXf}8h5Yb%`0vn3r((q?4VLW;KPN8WQdf*N;?`!~dF7pt z*)E3j2j=CUyK2i6ywYj&n+?`)OF}2Vo82aHSVvUR`n{Hu$)Oc%_eRF|7|Ml&hHgsD zlnCv?|QelnMHBo zp)Y&2FQ*^i7L2c7!+Yt5QFy?TYdPhttQ_m3Dyx)^tJjxv*lqmCX<8>KVEYgMRXadc5*`Y96qyw!#$#yon<)}loT zPB$)TB(Ch?Z*`h*UFe~Oo|RD3mWBuhE>11Zheo&G|1%f8IrGqgit`oz3;&;aDfifP zLCmM8Tya}A3kl86wXMt!EY;ZF_NqCM8N#ZP~%Wj^^%u&90B z!zjT`GR&PCPaHhTstVTbbW;i6X}05C;mnv2`_@Y@*A{&=vE9x4H#f6#^-V3#yzGer z0Xk_Z+itPUZVpd*Vph2$Y2OJm-lpwb>!z(Q-m=JSUGx4k8!v1*BK0Wj!|7b^Io>hh zn$mMtPhA@$Z@-`?GXH_mpUu~v&vBnxzkQzYs@!#1A-U^|LabL$aXT+_WnIsw?VqcC z_&xj|Ek6;wO?`>-ixpojkDpi2{^#)H%%7Q`x1Wl9ecfqeQWE#)Ij#b(_tZQWy(wy} z)wNG??5e6Mid+$2Vs&MDtjzi%J=c|+3npk4^94@Qdc)U$_uYe7zx25$sux(st~FoM z!SPJQ_1(`5;p^Y-P(=G*Q6Ll~?6 zOpjY>ZgS}8x&N)LRp%7XFJ*6ixv{1`^LlCf+f_>@?VoMaY9P~ha&mj%3L7ADzB^zy7TAb z$;s?H)w(_^%yOQ0x%!mi`K_NfmaRJ(-0pI9%_7TRdJM_oH+H*ERag~%!|f3BJB8|) z14aqku20vpyj-}w|E-Wg=*$nFI-k$4u9~;=>C1Hayxo6Z?CU&y-ahuToyK2&zuj@_ zogZ!U_qhH#U+YmO_w)Srm)nmu+y4r`P$$_hSFCj-QY?XA+R2^y;Ma5e&HkJIU-0<% zIpJx-*M#p#`yG`0#K!HLx9~um#`cGNx6l5NlfJ(qCOhu<+Rd@gD;VAje!tN-@$see zIqR+OO|duGtN!QEKkc8vfq`qnChx1s7rnu>@3Q9Quhq*h>pd5HpL+3?ieF$I?_M>5 zol4tl{?C%Vt^Fzd!&;;0gW}erYjif>*O=OQZbRq}LzO0FOR-JsHoZ&IpQX!T8FIOI z$r@pf*k`v^y0@p7-e+3p`Y*(;`q`qRzuI=mIXAxey*X#=y_a{+ioJ-xc>cq|jjq*? zHP0uu*GEmzU3PmJ>r$im3oPt=?JR6HHoMgo?Xtr2CuiAwygF%_cJw)4kH`Nz)<>CN zbl!XYOJ}xEJ;P7YM|a!j9KGS`zHz%n#M{W%chqM&eG@PUS(}$N^=0Q|mZRCu&p{Le+R9|fJqsJ-t1eaScX|Xx(S$@LIpHnks@~1^kQy%{~wb(DF zO-!`&%d|%!xeF&~nR?EhlsVPp-uUyWul4n9empk1;nT%)OZ|+)`|T}iKA#iZ^!3j9Z?QYqb)VWP z_2x_Lgw4$N1vhCQKY6}#(Sx+3?(%#8C3`S1#0MUdJF-t?7gvK+kgmxoM;Vdn509o8 z?)P~y#r#p;rL*hB!%iB?C<=NwmObFv@BBk-U+@0lA6)ZQT^aWYUH?$(G~vJCf4gTZ zzR6AIj^eCPXv|Ap*Ay_PghT0}>qaKw844WB-q%esxa*WCdbliQeR0PVi!!-3&qGN| zt65$NhNsmZX`HD1^54yNQD5X0FG{P(S#a$Q7Q1t)ZLRT(f1YQ~ z=FN-?@cp)I{p=SeUwi+XSZ-te|F2Fse{X%spI@?v_#NsFc4nXOJ8iu}>@COIh(|}R zc3r;e^`i6hZJD*ATyyq>#)Sk&YflYZk@uloub^=0D`VHL)rW*$XslXcQW`OD<=R&( zYL;y)6W%5MX=TRU3khZA&oh73&)K^)CnxOU_2uVZ@Lw#MbvImK>0ho(cDq>@Rh&po z`Q>F%`^T|4GKxi#zmdayhte9MZiDQXb53n7_F@;mH6^=RNYL26KKSLFd%pt88&*6N zUD982uQhyI>GycOZhqbUsgGW}MjIPT$5s4)+SR&Qeo`&7{gLMKwMqMvFLGR2F7eg; za=!VG!-@-bKilyyl2gZ?9(8H? zw+AJzeYK*FZMOBA^9w>N#5EGO{A0hLrq6VK?xRC5yLUV|RkZMcBgb=vIi+kWCMFk~ zS=P^8XCLm^wd3(XOZJ==S}4w+J&p%#jq}VW%O5mlbl6DbI-f!3w!yUtNi|*_OB&V1*@-4}h&eSB`Ca_qQGitYY;&MI-A1o~$` zubFhY&F9>DnPuxc4*P8|x!(8eQoc}xXjX~(&e=PvErr|f7TXldmSyU-WGTicu1=OH zF8;BTW${k6%X?WSH`L#rd~@yFkm`og_-kjcDw_1?t!nbI{a;%Cx3_$j?4m2%a&Lby zdo2I%Agi&4`R&>Tp_e}=y!p8@YWKo@d#|?q*W#WT|MO^n-u&IG(l+KF-Wn`Wkk$FJ zc+;g%#m830w}u({)gG*Ws5s-eTv3|1ji!OOwA7kzrHmaFFLQD_ z*`HrUOt}Avoq3y5`_>~3mS4KI-fO4~Ir-Kkf$?tY{p&;g8*{>sH>sq~WQ<-| zcca1b%4!}dgZ?{tHyc>DoQTdlpu}lipLoE`{rLyVE>}G*WiDI5rvLYx zRF2uf8!GQD*`gVy)ta$IXYO3zaZpBkeUicX2W4#BZ9OFs(t>a6ZuR2R z9_^))FE{8Z%U(-ea(EYt{2H)iV8ofl*N6u--9{BbAb=w-J&YODrLGb&0F|I@L!m~0R&aNr3+tP9Hc~*uo zL-L6YVXGPArTRpq8jIznotfTcYRr@NO;Z`>2OV{+nF_Q?1tuO%0^n=CdsbSv$~UzWp8_Ah5l%HauY zo~GC~yHDeg+tjqZ7p=F{$^X7`-2O~ea3ufF#a%{!BzMeDxW1U@`+|$R)2{d$#Z=wD z`AOpNTtf??M(w%A;@3_y`1Ka5a?Q7T*AZxR#=b3e~R?VOu) zY4G&uTN|E4UzM?6zK- zS@piQC))Xp*Ri6Il;6wVy5Ib3@Yv2+o-bP{_FLfcLW|D}Z*DY=c@l1Y zZnD(Za%P|ZD@&{|L@nOGX8qEAMSOEYLwIMb;0Zb-S7N>DG>aDyH9UA<6+^fw))whf~~HZ8pyB65Y1k{$m`mDE{k%f`+t3kH_I&EH8bsw zoXG~$1ycjoOQ-c8Ezn`;(zwI9bj#l)i}oe7G!HKTXTu>Un@I*Z(r8D%`1ZbepYnrFY-fA5=j5?AEUeU&`< zGhlA{;WclkyuZxzdVyTlsrc7*N?AXiZFy+TCLyN#yjnB=y5!zKeWRowZ|>b$v;6za zpT=`ezPgrijytO@HQVyGr=k5+!#{5O7k;1qDE!(j&Es2abb{|adi3uX&)-G6?wq#S zVD~D{*@%<*^HP`rgs`hUle$(Gk5!&Iuq(FE?jA6a$A4vKf7ZuGw$U~J=MMA zRq@u~$|`G7j1Xb* z6AGn_`^LNR*(w3tMi#6=KexJT{{?X;!AmgOo&*@Ke_H{p9QJk%1)Mqe9 z{B~hj(nIcwyZ`S?y%m|R|L)v5D>lLB9^bJQ<> zv%c}{+vm|WEtUK1P5lEh6gQu{I4ADK=7cLBww3s-p4oow`Gu)&bEcXoRb)i@1C?VF=|rSbkzlS*CcGnRGTwP`pVCZGj>^78O-opy?X1GX;K$| ziKz%n>iSJwa_mZpgp_0DmiV`qFI{YO*=uDl%6t3@#}zL3hiNu5FI_qKu);#nH~q?q z1Pjr=lm(7f!o5qbo=LD+cxbPo>CN?S3#Tufa&5WmmJ+L(UaKFv%$H*Ob9kboY1V=+ zwJT=|oPymK&veXK=a{*vf7`MG9iCc;-v&Ht(bFWa3e8->wpE#dXZbOykYkgcbWG!7 zuADhV?A(!uVz~m6Y(kf}Y%bX(-*&R?g1t;PqRrM`GnZ!FZk(}0Oggx6n+b=dTj7z#hR5#p zEHuwtyq{ysi>p^2Zn?I+IQX`i54*L@!aw1^Cp#D=uq-l~nUT~mY2GD=cQZ?DMT4ax zj(y?Kd;GBR1Mfwa1=oxwR?W@4ut$HETvzMy4Dl_cPSU>ot?daJ0ZaMYW^^voOp2=2 znA)Q*xvg!@y0RBb)?Q<4OMVz~JSxL|>$aVVew(BADoJ^&uI$N*SSxz>&1222)~uWQ z+k>MukJkHrZk)y#;Kci-_oYZk?B%8FxYDn9ZLMy-b@TrH18mEVUb^){AtXEE=FPo1 zA2+dn-aWl^U5K^#>Tvz&4H>D|l}i6jy4o$aHe$=B+j_T-zH=0Copw1yN5oKoJtTdG zNYP`))82eb4mmd`uaRH%XNv#cS=v@(r|2* z!PXO7nG2haedkMOFy%4k44dH|5Nc(u5%5n&yQp_%+bZRUHwt!$Z!wy~w)Ry1-39AB z4Q%C42QivR9%7u#vHGQ=uF%_lH@#&sta@rQ+AeK-xNo1vRKpCvF3WXQQFD^^q{=;P zay+`|qlPMnRpM8-$t%}p#%`R+xB1W$jTdJM@^xm(E&0o!H}`H~K%RtsM^=WlzoYsDxv5{} z@lUgbE4hOo2A<+JaFh^9h?@I`l{=QxPMec&-#n>IEve47tLqFl_nMrED4zA7<=wG^ zJ~r1R&Z`_MTx&S*p2)JgMU7(r#HD+(3^lI)D-y1{n(<-c)wTqUla~^t8dM?{aBC`T zJDjqQb)9|sLWgg4;@o1H9xlz(>@I)zW%tpa5!doK@^bOA`A;5lZ9bq-{WrTU+IqeI zTbmr7`8$u;oZfS1&ygC-In4pqXM&E{Da%M6`TcRR+7Tz4Ujk}J4;gncFZ|E_c%#H7 zuGGAg;*z4 R;ybTf*vp@M+|h+$)+@0OWUl3Jvo@0OETlIoJ0oS%})r4P9`MZwtA z9OXuo*hzQuniWKjzpv%e+urTwuqr#rXH~AS{-nB{?^wmZ^q-ASQNfI_{^?A2&&~al znQP!b$8u?P6pIz>qfiC6Kx>86Mxn+n&R!GwJgS#0Uan%z_K3%HiAqLEG@Fn_Vz(fR zyA`Wf;OPMEVD~%irANvRUEX#`>3Y_t=-mFeDG#}#5<~T8$UZx4AQ3-VWc#7NiW2{h z+;<0-+8^D zzB|q2-s$)D3EVqxfT9R3vWyUs6;fG{s(}5HDihFsN+_39g%qVGl_qDTmiPwiI_2l2 zaOn@|ZB?LKj6iNQHZasPGP5*|RWLSFh&D4e)-yIRuuw2iFf%mNvoJI>0pDo_x^~LM z(#%l7%-m4V*uu;_R>9N|bQ_f!_);OTas$W(QwW{03T74x(Wb^mdgg{E77C^ohI*zZ z#tNo}hI;0P79cxJ%^`OP87P<<80uM?8XGE@S{mt@n;V-b7+G5A8G~-D0=d%2(nQbF z$k0;3K*7|~K+o9NNWs+1RL{~7Y>I_3@Lni&|G;J=m0($oUA2zpzc84==s?Sp0c`_^Y> zCMF(J+Qb}P0|gwDw}?sw#1u02>MAG+1PZuZn31vVU_zO&>1m10Sv{Un$!=$SzO5+y zG;4;BkZ;fJl#4q1BGtMLFa4dnYTpB+S>J2-J^%gL_WYNxC3B-oSG~G&b9JZ#!-fM~ z0*MZ_&Z&Qw7az-G>i8wi@GdsMZvUs_`;5aFc<%5e*xK#ivoy|T^_I&KJZFopUJIRa zyZq)irXw|c3U?!H>jO_;5&yQ=`SDE#wH52u>G0RzT_X6_pT+iN>ABnUU-B&fnphC} zcGAw($NJ7Pyolp{AYr_HU)H~5ew$Ekfqn80|8^yPR0=OWe{emsMfjyz##+y&U)O#) z{|fVr%-Tv`gL|AgqT%=ArR?6McO*wM+H6ibQ+Dm&)9cyqd6t#Sn9RJ|?2`WUi@>bK z_e3O|Z2EYYvAfSa6RUj8{lf;iU-MYBM4EV*UQA#M&dl2P$MhtZs=~~R?q`&CNj1HH zDE!d+#Nm2OO%OkyQ{`h`k_ZODu-SW*NQ`yAJ@QNK{@Y3IjOFkIS4R0!WnVKQ0 z_2TdD2mY6G4y7M5dn&QT%bGK)^PlFd?X1hXJ}(H(X_%O##WN*t$;O-sqWUeKpEL6H z4+cnOH+pWlpTAIS+veiWn=kLYIn(<5k)P={v0M`U-{U!s#LGS3sI+6^f=%7qyBj~2 z%5_9Hd}%Wm)IYwYWQMjKQ{)vn)d-!bPQ80%xhz!<3oY6``-Ozf>Nswum#GpD&!ow{ zZ`$1?@11JU{6R8^jc01ILg+33ZxiMIM6GGA=FEB`dBNY(u}HOU#=n_wTc5Gli#~38 z*~`>%bF z+V$O8ud)4Do_d9&6?5LBvuDrc-+21D4v8T+l*f=%)q`;yB1Hnc97lOX){?b2_he!<*K`uu77zWKj3Z|ZKHKhgec z(!V1)v(9bUlFq$z^AAHmy$Gl$@?SKeaA-OX>5EPaf2{tbNklct`$3ob_!z%hHqFMV;ZBlJ7YD z;4Kn5Vzl>^IqP-J(u1W*`B}e(QUuO$tP}2P+PR5OsNd4Kx57C)@J_Vyejedg#Sa4A zrkmKL`b{r$Y(EsU?scb&x1swjmhhr~eho{`v~0Yz`^r6*>s-!F*EiU`G}V|;GQoPNxIt66}d~w6?Wfas5~{*AkN0;-zUAflMm`NZC^P558F}QuFIF^iu7*_ z_^so}^rSQXfJP?Q$rU_Z9{c%T+P>N^`fT#PsHs=C^`su8F>CXh?qgR%3|NW@6UncoSa>#P~xHs+M+jZ`Lnp18x<#WsV z2aSG_8|EB*WOrVAi@W;Uqq)!T1jO*qJ7#(zp2MO3rNqn2{~Ty?TnF zgQ4iQRr;6yO)Z!z$bFe(oyrcsUjpw0?V4sw2<)4Bt4i#3?`B@nwp?01f8WZ# zzn1e#%$R4WQCtxAZNkEEzTH_rIL@C4wA^`b&!47?$y~?#r0o+QEm?Fjc$uHJ-b}m4 zuV(IxT$=jSV0V(=(>tqP&B}gu^;~>F`2GzG3j;4ddNxhfdwp!#-LI#^?-V@tHNXF9 z`L(Irx2@gv_M7#)J+I$YhuyEfKWppjcl8erw*TAn?;&r$&957vgYgS52hW~=@6(yh z=YMX`tFHUaU4Qf9zRySZ|9iW1dVJ;At=I2Y6kmU5oB#Ij@7w>kzMotEeLwsE9}f=q z^U3`AG1+;(oL$kE8--obr(Z1bKImNfIADj|^vAXmBHuWwTMs|_nl!s&_K%JqeJNAb zJdBqHy?iIP{mYN(-sP@4t9A&*?u}uNkzBWMUE;cj>te1KU4Oi;cbzV0kyDCOiBpo( z`j&s3n}u3kR1Iwml?=5uMknNZ*_0(f7^HK z#%;gq<7IDc+rQn*dBc1E^!{W6i9b9FFB(`H8e|tFyC2`yx7O#cthZH^)huhV@+Y<5 zPNdH~&a>E}?DR&{&CNj>y_y6VH`8D5PHu-zsu==EZ=SNfp>wGz;K7O_JdtWHq zXxV7y{NH}hSNwZP?;YVL@18XRdCyLWRq*7sMHdLHX`K8)eFt;c!TAq0D`d_&NM{uY zuW5V#kbQ@2UE9=*HsuepYue9#(El;bq{NOd-=ot z3G#cG;}6FCV1Lv8{ek6*ruz@_esI_|@Gq3w*O>n>YzJFbg4~(*s0aq>1$<&owk8hB zCI^)wc+P$>^-R;$Y0Q3T6~XrQ1BXug>z881%yZg;KPbr^>A%<%#^WsArSVX`g2Vpk zVU5L24_7Rf4Yg@<|0p4My#7(TMB~I42Q>c3>~COT`RHxasQyv=kJSEy`5#68nb)=7 zf6V_w`5$-vq4}Ps>>tGci26UyuVAY`zFDHz@67>*V+lHChb?}n*R+Kvlz$RCASa=} zK!IbC0_Ua$99p{!7&U?tm3FcE8YKGr9EzKvGlP{|iR*)9O{3-w>E8+Q2en!_%@kuU zuY7RSplJ2?VtP$;BcL+#t5EM@o);pG#b5M0d z@2ZdYoaBFSziBXU72F^j*AQEvyoYb?1x22Q-w*dcHY(8CpNq3hPJT)*w^wRMNzY}$9t@}1>V zWA8snmcCiSdvkNw)>X&X=a;YB|NYYWQ{(%gnROV9G%x9ZrE z?$y4!&0T$}S5DRQzof*ez8k7)Qtyv6Rw(nmtl{MBnI8I9ZicV^KheovPVe<*$ZqGg z$*f)eyVc0-Q`U2jDwnLJ4@@|CvS;mxyI*iu$G8n5|ZJ?>Fl0lX|ALFyNY}+inRFp4|zu zRf3a?xNb~1vFw@15mg_5j@o|>OD&q!El;kWyM3CUY^+V|x1BoKidn}3@~3Y%`fkIf z9}!|EVG_36WIog6){TZ|FU+vwwrW43@+sz(aE9c)qgx~%e>&_E-nsjd?Ok89Co>}s zojaCt_G`4o%~*>}H_q$Z9!}1kwY#85`gLh~`>q35&2I6h_{kN`dDG#z@SaVVO>S;w zUU*z|WOQY9!O@)U-sRhKb#-!9f1kE7*W07z?LqPIqd!0A&3Q;%S*p0hHT+pXgBx! z#pUvEh2|I~8ZXP0+xYYD*+*AdnIESM_^4?p&2TZ8m7wJ4`|iLGp{As!OqR!-Y63mV zCzL-3wX8q#)Uaat`^5PRjz0)I@YJ{5%=XTgn9JI}@6BX$gJOAi>|ylo4v2pr8NdJE zy06mL_V0=-Tr0Ps>S)%*;QsR1{;${9?#Me_`tNmLton7`;~!E#o{l;FE=H|2Ub_D6 z-0NNQj#@a*Q1F|eb|NU!yS973E6@4f^S-C|8h?Ly=XsH+hvz~itwZZ7Hyv`{Bix%e z+x+4yUxVt{S>|yczfIV4m-W5qgSX3$W?Y=*+>KX!9w*x5Civle;ITF@AFV;x&J z*WX)5R(o$#x@UM-K9Hj|*zGOLkq5flzwdpLE>fod<+!ij$MrAm_FkCp$5C=$ig$6( zbjG{Ozq@h;F5Xa(bTX{>S8-a~t&F{OKd!CO$?w$``EzR7^E_FGujiK^X6RVkzcPQ% zgUuROV)j%&UbJsb+0v@mdz>3t_%wUpg$URUvn1Pbyhi@x0Eu-FbmxKTjtG@sILs6LHhICH~%wV z5vLj>(R-eI`y$TC3#VpH-~G{G?Pt@gsVhU2@3jYK%$>eM%&T>KOY#fh{3&q@yG}a2 zFAwJZI@iF#%46y|;fjuZ<{W;n?(4k~Uv*^vp?s&_0;Vty?G;+LczP{nXx%$^{`-;R z-(|di)_<|#^beYQeBRZN^0+@|mZt05F4zCwk$qHt>C#ITZ^bt$r^zd(|o-h*j=r+&GV5wp!I(7 z)iWR3(u*4Btmg>olx(#-6_DCvC)dwX zgTVbmLG$w;Gz7^{`z7R3Wxzb~R`6%0XFD4gJ2^HRtV`D1&HR;5X2SQmCnp@TvRho0 zk@TVE%alKME(@=2wwHf#t;cEO(o>gm1YfrGIdR=RwECskUZLv`)pCw+|DZ7IsQ0%+ zGdkFn{>dqJ>3;HhwA)ZjDvGu9_w~0t_Wpj;xW?{^{inG~H{O2Uc&?H8 z1-Do2b2%Y}9|3Ql`iU%(5tMux>8E;-wMSfDlQ)R*&H=9YCy=zpDSwY2_0^pBqh z7&H7kl&Y8I*Ek4RTQQ{{jlJ}{aXDZ4nQ7(o8YZMYEx42;bLZJi+pxr2B9jHTzFTq3 zDtU9#=j5eY8ElM>+gqPneND0T&cAy7mGoEHUvhsN>YU79uQIAn)7_3uTdmmFu`*H`=NU5UK7>B*U% z-{xezeRqasx6)4Qr@QOx{)g{nKmX@Xf2i@jW97At&gJZ7tnZFg7o6h%wBmb}KbOp} zr>_s3-1x%kQlNdA);~%0%bPoY8chC@^OJXjgUFSNqP<#OKdqj*FS_31)MCEV`@tr$ zwwHtQwy3p%uX^@KD>C3iF;er1*Mle zgVt$TKTWapJ~use!s9OyM^iR;Ph2ec^u|TC1;sNf)+7iv7S6eN)M&fp{gwSUBHN}+ zk`mewZe%#mZ_aJmuC9y%zbe7+?9*Z@eIFdpcw_4P;=hA%p6%|H3$7<;%hzt}f8f0I zv0`OZ@XFaO^EUHaXBj)G(_Po1`_ zhvfy?s)KROwy_jhq$?Cvk- z&Sn1F4jtm^*Jb%KZ;8gcBXKjEtJIcQv+oqX`sASG%ZuMM+9!YS-g{@AhPzwUvI32d zD`%d6M#U7uTZZhdh*AwG}#R`_EA79+Hv_yE(vDw#OU$Ur`DQ--1eDkF} z>C*M6t{YB03Jn6Y^B2xgbQNi}><&37lolAud1IsV^K%-h?7FcpqCLdEUDmWppY&>5 z`puUz0!y2BUsgV0ZeacUn{lyNee=2l6EA<~JvZy?p<4%{OT+yp+?N;I5GQI;VDw@A z0Xf6%kM~ZDjyq%{w%6nNBgRXWB72@ihNkA3ZJ%|0miesm8B@>2C^*_JeX+zN|4E3K z@pso#N=x+>kKTMFqHwY&X`1DI9R^pMGh9!lyDv{l)!epb>6Ar1`CHd-&EN8V!~2c$ z8?+rPk2`Uyb}w4EGlS=;rkhSgPQ;O5?z>{$GEo!HJ_>NwY+e1^K=WgW=Jhg7?vmZR zgE+2RTsHmBrz+f<#By47Nz4ZRMG5Nrtk*xeXFc!0#MSF&TK?a->vH2`3z5VBwdy`y zKdGO~V`E+%p6Bo5@Fw9ui_Q~fRh52^N6*#$t5)I6wQ`fU$+*gFQM&$;EuX=sOca`duqj4#wSv6rSR?3@{J}J*q zj#9qnrXl!3qW^KI_O__&QRY$Q5#J;HBSf>;tV-B?$tyE!lkC(*cCP|Td$hXzwnnTC znjPwCdekCF#%b%~qly+YRT^kxJd5jky z{_=NO{zAp$#p|3W2*}EM9gC?ub4+iu02T>D!u&D7W-70l-GF) zZfX^BT=HktZuX@m#mA1j@M?K=p6}D|(C=K|H~n$j@sj*-A3%Z z=N4r+oqK+!Xw}6)KBFy9QjDy*l$s+KOuy4Pv2UHj`joZ%UrxMrTkg&0l=%(`wJ=G>@{mxwLQY%;Mx( z_BV@a3J%ZjPG5VY&?D@ZRd5vxpY5*TUAwkytoyd)SJbEbVW&7EMM{+vu1{WNDR9g+ zV_M-Yi>pseRA+DUIl3(DGUw&>YP>!Z)4D@ax(#NZF}r$6GXC7@Se;EOH!Lr&y31j3 zF|e%g#wN?d>-)J`PMN>ARk{;>anj}QIRSN*5y$ye-M^P#F`B>7;nT`Ld{BS&3{sE;Z>j(J7@3v@84g1x!rW1tFxqvciqR7YS)x{6_MFzf4r~D z;Ci}d>Lshp$xA#8@?LUf!zdi*rPyY5rpwp# zN}|V;=2me{<-?g8%Db0YzP9>4^X=c~?<${eFnE9Mb^CYwYB#Uhmp>id*Jq_%_f(L5 zjot2Z#?5nPNTKEbTgUm(MkD z&Ew4q@-62M)$_|V2#1t7Y*~9kBvNIT&Z4X<3+M7|{_x1!Ugx(`<%bQMSy(--`~ym6 zOq3}9q`rmKul9TJ1NJFRzT4Sm-D=6Rf4tzpK?ehl#_xp*?g_QO)4puoTw!Bs5G>{N zJTbDgOT9Q#MQ`eb)xK9XDmKjE{>jBAyVX`Rd_$ZFMpVOc7JfA+t^zgRz-1kN9 z$<-PBN%`@zz%)kg{5x}|-7i!whc4IJX1UF@EceZ^8?qapMX*~O+dg~a)ak<2XV}lE zpV>d_QGdjrqJe+9UA$+5R8;$EN0*jF6CPe<&SriS`=p4R*@ zT^Gwms>-h_mS4YL=YR6k0dskqjfJI-7j4ab59gic+i&xp?ftx$SHpYH=hT#6ZEjuv zJoLL<==Od8_AS5bvx-^s-yenWN$aOpy1jU_Z>InC`CCi=-dwm+>SVZK=dwcGf?4a% zK3H;QyO@ec=b1hc1sk6<({)K&O-hD`l(ZIl)k^Glf3N1d{9stWyRaWu zinn^XxtT|WF5))w7rwbo?6%i!wcC!jJ#S~umRvPe%r(Kjn`4E~1YO}Z*PCnhE?KoG z#VlF4!&gYh-1GFyONm)UlOAS;s=TwEwPfZ`ecNOHcJml{gxOq0hS2%TQ6s*ucoS>dmVI>W$9E%v)E;Ii#|+&Hr;$@y_Lq4P7gwTb{~Z=QN*~lG2si z>}izvVNH6@w6><4E+f^MF2xbsH`paEn!%*2z!74YTx?;)FA}BFGWAQ|#)yTj>mnwV zI+aP7wI@dGsC3$NMTc|J$&}4o*yHb>>(dr|R2X^Vp!4^wDcot%Tso>^f;x#o6EhZn zIMMI7RH<0d$~f{)-oL}V|8E_y`_Hx0KYXP}dZH3fbGr-GUG+qQEGGe>Kx zeRB~#s{1+5zNY+UUZKlz(L>Mwyf-d7Vi)`WLY_s*zs1(y>SfxbcQ~vIOnji^`kRwY zZ{ppuODA07Qy*}>xNzcsdb3Ay?uT?{b@t}j7U%wnw@8*r?qIQfnAvw(cV=3B>I0!2 z4`TaPUs=EKUw*Wv(!@`4t7k zKibjyOZm!zM(+u$>JAoaKMsDK{ou%Ku}5c*{rlb#mh(X9%f)q%cAUFr_@Z=MpQHTA zCoAtJJW%mp+x3|5SpT<gmGHd;SEer5LKk8}1NIw1KsqwjmlFVn437B8=x%zymX z;;%8=xfkKaG1mU)ZdDdtJNG!=y1)K)<(F%pU#xw8@_yC{`{nEEJe_?*zbb`$x#nL# zpy9JM*S+{>-p1G=9yMJ>2`hQST?NXFaoUe&gRQXBO${km@7H4ev#;knj=WfeidinGF%$&IW>niJ2dcEgrO+EJf!)4=2od?Uy1&<2-n&(!_-LYxXKI`j| z4*r($M~}Ln&s3WF-5_(CaL8r3&z#W*e&2ZG_V3z>jb3{HcTT@kt@r$zYyMM{O)Zzh zKgeqTtmCao;bWX!uX|8w(W`pNPUG40j(;yGJmuE>-ffcmS=MU@%hHNwpHC0{EF9OB z{w?fy<@+T`LMIv+YnIIXmG$GF58Kl}E?Z-gLg#F2{T?2dP@B_l`6s3H$ZsZRuBmgj zs~t9Tj?sT)G$Yv~OYq|EANK20W3GR^c>3h=)Hxf2ul2SaaZ*nZ`Py=JX{6|$_RPwC z3sbjMdoUOO$a?-h?wn;t)6y?jihgD7t7l%WX|D4-X||84N@!+8&~g>=OI}m_VI zI6?|Fu`9-}d=QkHZC1ASx@YLY=jKKGS3Z#| zuq%C4Sr+6w<*Y%|k^;Zx=~k~}_9ZRvo%{KwcjU!4++PkXN;uJ%z5CbcUeoMdIz}qr z*_`Wo??kWa+4xDX*L>dgcNxjw+1|bF-odMO=mxh+)y~kFx>r6-)mwkJs@ZH}*t%yk zm&vs`N!gzL^U^;%E~oqN&L_`>eNS*ZU7VY_Z`Qrb|HNl*epY{CPI70{$$Z1Lv4?Aa zecJM~=DnL-zF6ViKY?a{yS^Cn*Daj7Uuo6pV86%t)8+LagzcZuk?a#R3Ve9#(e~HyJw@4k0 z<=qGVGoHF&a2(5mL}SE)M7*m<4b9P4k7l(P2(-TcEVA=XoX$Vjw@n<~Z=1ybF5Jfd zK<%+vKyt@T?)&=+kKJ`*=8Atd=j27tD_i{6UYzItK5vq2gL!6@k2k~2k%rmRpA@^~b; z@TV_tGk%9>*1)?m1MDi8J>PG_uq=k>hAYO1uYHpZm4o3EfsW4 zO-vOG&CSdeER9W2);5Bi19duVZKJudsihJAwT(uGW@Z*BIpob=_KMJ>yLS9v8(IB6 z<$cQXHx3^9IUX7dzu)V2U=S5$3317JAt-7q77+5L>qei(wq(bC)`Q8`T{7AcjDpqRW?x`PKiw@BIJ#=8rvf#+%c8|2~_YKWS^&5uu4HGu5}1%qczB z6Z!Vt>km6l$lAKD5v-qC_N*uNw)`sL)i3`l9qd0hr|jUGMWqKdV@^L>x5)V1EIVH( z+vpCRDz}t#i-OP3sXgeatlj^_>U;d2XIFyLeY|Ic{E@8u?!diB{cXm-SI1kz9li&j z+wtMa;<$geL#Noc{b-q-KDQ|Cd7!$!(9hdCk5(_AZ@>G5p8t-fPtOJ4Tm3zgxp~?9 z-PyBG+;N%sX0q${yR-g$D1Yv6{q@I{eN$gmRSRo=dKPM4FF)DT{Cb`2kDGB$CV|hd zz5Mn6+n-&$ML7?G*PZA2^JeMw|JoJ4%SE!e_(W!gQqiNnV&r6m+kUm{8=re@5;B`Xl}gc`KHYa z7;ZTki8vN#Fxj5n>SHN$EZA+Hp1XH;E6>*VQ*LJMU9#%vB)-a)%U4P~!%{cIbtyFe zQo1%(Na;>Y{APWV>yv!<^!$l>_51V^-s_h;m%k`_llsX0)o&wD{-AR%m!B2g`Zh6h z`@3qkw;7%{+|rbbdv(ln&o?W)^?STeoKs!kTtdzzj_gl+4NoiWpKn(7u`E*i>?vyyS+yzUVf-fyAisydqo6eouuXbRYiR;V` zd8T9GXJTGdW;D!XpRRIw+t;)=cPle|!Zkl0((FAE!hH6I>g+;$mMgJI&lVWYJZG1$ zSEevE^yCb=8STvmj~}%)_j+tB>sY$<-RhO=H^$_Y82>I``tG~S$*V!HyCtRP{+s`# zYO~&_e-3Fc%(7NSNz`tA@tQwvkMZ32GdI$$OxG>R^IuhOEdKIHq;i!1*B#Qatik41 z)f47CIw{I~>H6%_+25oi+w^2Svh$8@o9Gh1MNjmF>5A(I^Hp94t>^XMmLsBnH9Ktm z?E4;Hj!O62tM2`JFXZ(M^ZoC3#V0;{q408D{_()>xvno3^SuuZPh|dPal7}O`JC5Z z&)tpDHZ7ie^4H9~l6_0LCagbGc~h%=p5eAvewoY6W>sD`JH6?4#pekpZR(UOx9~@& zyz!Xz>qfM?-O2B%9>qCccByw4zU3^Hm)KnV@_5CsFM7ag;?WOm6 zZrm;XduGu6^BfP2jy?6d_OJBVG|?5oODnE%{WIlKM9 zjk7Uj%Gb|L^K5%BamY%}$SUC)$Is>{b?Fy}=Fd`o*J~1&KX0$*$7_y3&0iCay*IBC z|M;tQoq1028@of2S|tKl`fl%Hn)<`PFx|=dOSK zu7552;!dk&&yQ_;Q+RLH`{Eov+uCh%Q&((_I`!)M&)qLhOWeBK`-S(6f6hC;E$V6J zd^_Jsc`SZ^P4>%gjW-95R{2`%-!e;mJ6R{c{_N+Rz)6|?_g06^*t;$yx#Zk!^Vh%M z-xRj?UzDZu^vJd1HTy)X_6Edn-MIDetG8d|lBXB%-7)XI%xlBUwSoK!;f=fJr%Bcs z{7#ABRgZU$o_o?FWzp_+XYVs@Jr*`~XLSM3u|IP8@*Z2ZT{^y7a&r30Y`K3PJA-Q( zo~@Ykb;b3%otuwEovg|J60xCer~MPJ<)_X}zj=D@-&5yrg{7TKs;=q^Jw8Kiw%s$S z*c$I!M~_`PJa;?qmhOK^(z7PnpPiNSYevG}lw+FO)9&jfoPFx#{cH_y@uSA3tQ8!TUV2 z>$k~s@A$(@<=*|9cVw@6<-?V?mYH^xv%P)oV{qh@_Vvf<884rnZv33c!DI6Krs|o8 z6@TUlM+Dohc&vMJ>fN5p9>*5@L`A;8`0ZGYXYKsM6MJ^+mA8Ibk)O?Z-BelQ{&@-8 z*MWWY4d*re1KpX|9G)3`w%}vum9;{JmnLX87CYS4zQA)&eXr>!-xQ`I{XI{g6|*^O zJ~U2X-s5f~o0GwJf8w{r$7kRBnw`k|aBhL>9j5;c)*kcu?5j)X^X4D6yO6niXWR3* za>hG+(<^jZekaTe58HG+N#9n!>fp|}x#0&Fe^{U%UiowHKkJscW=EzgS!Q23`}N&l zjdg@lYOwa1N1Ci17&m@k@l}l%z`C1rz|8?@?ba$Kord$8*F^v9InJM|~PT{(n`IoP9uX#WJ zOcnplwL9LY7uR*HFkRCB;Z2;_wlxlHnbwOghpgf`q<-SSd=2Amg|=ohY>w=nQRDjJ zi};N@I+OLM{{D09TK_Sz$h9?IVWu2&UJ4U zOAG6FSsh-vIXc;W!Y#K|>KWO~L|)s?{Nrxm)*r~1Ztyy_W?Ji)^o1n`$=9Nb-h@ju z@4vh)@y&MsN6Dow_YThcAeG+lI4QILP@G)f-q&7#u6;(6OAKLzeB;g4G9_UQG>Vu$^-|yy|%4G+|zTM>Bv*X_7NoVi0tWiC4ROrnE^_cp;59N23e?Rc|$<=** z`@V-CdoS}Oc*B&}-4jG#oWK0!dd=Ib`=@+J;h6ombo$K6?V9DLPc^p7E{ls5zjV#+ zpVF;OZ5!{M-&HZEYMq;&i~hO^6W&YL9P#e3cb@y@GGd zpZ(_N+`TXKX}8hA?|VdUY&matdCup}$zQKtU;XtH=gDclztp8lmoqJ&yvy45c2n%C zqtmYbX^5YCX2z0L=X%};ExDG>esk&%rY9QT=0_h1``fX8Y5eTW_ePi6FP*Di`P}v0 zcj5nBGktHrQ@*cP^Coc1e)*U26XyBse(!p(;ZFCxI)`7iFKXMKNW{O6e4M>>^Y`Vm ziao@y=P&ZLv!43v&y?-6KFxdj%U*MT-==F0d>`36t_Qs9d{I-LDK+`lGKtvKoSFAi zYNm&5on%|KS=0LPt&_I48~BgLN&dFEU26Av>D!B0;q1wmejNUssh)kH>gtX4(|A*B zW7qI5o`388Kkn4mTNganQMRqUJk2L+)nBdTko0%id9N-$KbfBS|MHe}{pVKu!_d((Z*SGv{OQ|orp}*G7qlT}$I(5-M z$%WJ37hJF}Pw6k6ci}1HzK$uAw@-f^&hh=U+kx^4$_wU-r`Jz^&9L7ouT@!*VY(AX zSfu_5jpDyTpM@RfZdZSM^u53zzLiVG-Ibp&{=Rd9>mHGryfa&@o%j#^{AKm9?u3j- zYv2VR=c@b{4m%hol~_)Ac+$Y+>!Dq9luk4!X>7@x@biM-zQy^!m+sygn74Gz;v3Q}A{zn=L@ z^d(>Fyf=?pzsi&`*Jw&|E3dg$)OY3N-)oP5oV8A6ORk^smGR%HD#_!Ldz!!MCI8p> zysNQLZ}P?)lZ$Tz>uu^=uy*uC>-go@lJ(kV6VO{!8k!vlpv-^t7Zaq#k z{i3?>fa~+lg>}!O7+;>RtYM3vSjM=e{a{|q+~i<2_})-#^WHbk%X^m*j=7TH`L3P2VLQF?&tua`QPWF4mfTpA$6yxi0T@ zvtP4h@2KBxkCM6qQmtI+`)}jY*Op3uoUnQNzjsI2 zbB=AjJLl%4wd*_fpZfVtG`&A-S^Yl8KdR}{b$9(%GubtZ@7&H1*=twlVbS@a*{{_u zS)u%t^||0rMjy-1?0gw~OsVIOSa4|WREz30NzZkyAH3OEZCMo)zs2v-obQ!)^McPg zf0!8TxId^=erG=S_u3};s}uiLiqr)ZzgnA^J8j1O$eQCf{%Hl@nEm(3`Gsqv6U^=M z)|F}&N$$H8{$th78;`3ao6|R`XYa3T6A3aeI~bW4tLFTH{ph^2cXYl;ygIZ?wD#7ZnO}R@bnos^i{Bd@O@A0K`t>UN*`8YA)E&7q7S#8ro|`Z~ zYVw@-8i%)Ku3oSF^6^ePzv+%OyUguo)gSVk?qFXs=Wf`>M^`?+Gd)+f=!R;sqV?GB4VyXtOyk@_D}lY7@@)0(_xS1(2O=WP$2Jjd|| zLsMQy@AQ||9h(mx_nTY4d&vp0?J}?5-}tD%bou&#V?X4k83>gRCBTn?H_PXW7m4 zd_B(z^|d#i?40=8+^cInuic67H%w10e6wuXn)f%0wLhAF-MjJA{^z%3zU`Lwtvvc{ zP369waZ5f=zQz?La86|}=O%{gP1mGO{t)R*Z~v-d`!43tQ7Ccb zD{qVX6M4im->~}-wP?1NX*&X{Iy8p{4V}0XN@A3UNrNS|F&IuLjBaZIqlV(mjCH__btTT z(cHrI*6(x=!>7|8^Y8Twmr_&u^!2 zKIqF&){^BeZL9o_v_INk>oM>1&VBQC$o~|3(^|SDT`At+sfYplt~nd$sJgG#v1IDs zKj)`L`g`r?f5LlgHPhC0U#)D3FMq6-`?PV#-Kl?{lwR&nIwx`TPP9*b*T?9ti^1RP z?0;9gKUyzncU?PO`+Mfk#p3@3ZEHApw%Z)$T>tIWkG2nc9e)1!eY7WcSE=~(d4fh~ z!i;)~f@f4u7o1c1ZBp5ml1;`-HsAHMns{yUmUFLEGSkni?4A6jsV2V9TW5{x_b0cv ze%SEk(_G!X8&|%4%X{hG^{897)_clpe3x~*#qiR9U){HfSALp4e>l^qWplJ#>X(h@ zpZORv+3t(!b0}N%t}m-K)7?77T3FpDc9zkhRIc}zW3IoPd*S|ROY8Ecjcc+G-MJi+ zoAh_r%e5Nm3AVpxNy|A{Ht$GZu)e+Xdppamm4_w2?)bSi=i$@y3iZP8cb%JmFk)Gu zt>`76iq<7J>o%6oFq{0WX4$u@7azEu8-L#=eqTd3`Nmnx>&G8eZ>*HQcy;fMM1EuU z>W#c}Q_q6xq>J-o5_`PEDgpDr`T9l{oalOs&SU;iUli0261>d4lj+LvuwPuz! zYWQj_dUNJSsfN3Gp7#!)u%5KbUFe-tRCG%DrCk!1O~&`HUpn}%PbDW^`sS%6_Rnj4 zBb(1g{G6D3gE{Pk!jp)(e$iX>zAQg;IP>|#Ez9jIoli{oSviUCe92O_y<9)@E>*p% zNjzV&?}M*_mGuOhs(DovJ{Wicl|}ZQRQKg&+s-&fAKRGQY-w|6$y$SlKA(3V z5cT+5eTrxDHB+(w&96PL$S#$eKKa7Q$rF?NqP4nhCf>vUu8+2o7120 zuU+_^XT^?Hje>bGr+>Xx=G*7DL*4G(#w+GCD~f+zy>lXeujQ@mUj-39N!GJ2q_6#) zduGqusgoqOn7?Rk&!|p0H2pGv&V+sOX41Devedp`o4DrJi=W0Xwa?rsHG5oPS$1!! z@z1Zj+OO6+di>RaJzK+n zeK+5{F*1I`GrNYx8-0~8e4qNlSNhBILgm}V`=k!syBnoy_}zQLQ==PRlgs8ztkS;i zdTsZSM;F5DQh4`t)fel2Q*i(FaPrsS&MUzxcRzAh-PG=UKQ+(n-(1f6Cf>bl(*Ib> zo^oU+#(P&-dpF-Q~CV4zxmIfz3Tn_`?`NsKdaAQ zl>Jq?>)56(AB)}Dy}$L&`1ASv^`({P_en@rpL=KT`MG+w?X$`|GfL*4{=GN-(-~v? zH?bSjbM8DpGwX`|{?Jo)lj<&fo;7Xjp~%cftNA|4JYV=^;itdue~b4mpC^8`$2tE0 z+0Qff+uy$t_Ic0E_ph4kw32JW&;SQ^D>~ZQR51 zPFwjq=T>gAw)`DpY8BEQv%Hk^_hZ|~tTTBJt+kAg+V*JP>N&CBWN)lfe-QZY@_Fk& zjNdH2{XTgv@3-;0+fUzo*8EWX{{Eb`-{$(YhAQYTSbufiE_Nr$lD2v0;#N0rzWeLW z&D7Ax-E*G33F7{soVV!Q?FmcIrCq&iv-{ud=WV8C?i=66ovZtu`|9t!UvkaUznnLY zWUglL=J^M=o=dBq_t`n;&g=PeKObwg zyZuXYVnp@p+XC9Z?awT(ZTz{nVAbE1o3H{*4y+6o9E8`_Vb+n|H8Ocb8OEX z&AXqvrP{FglZ6c{^W3!oALsvi%WY|D_b2>*_nrTC-*ubV6Q*pwY5v1u|9=nm*QFnw zKCkiD|IBkiu2RJB*nG>M6LQx-;)zj-EVKVosQ0;a-A(zn!;%F}p%30P@IMIt!M4x% z$h^w`=5+w!gSM9G1`=DIm`g_AK--%T(_a0X4-;t^IM{JHt`=90ai_Ib}Lt;PT9xyxVJe?NC=V0wgIX%I+-?*UmiS_qS?m4Y_&UGr0PrNU#_7t!Xd*!m&d8bRqfAUT5{;F16?Ylqx;7x_#W4E76%(gspEAvp+|NJS-(2M{I97Ra&+E(A`A&Q6ZrLhkP@A`Ie&hCS7SV6!oxA>u@AH}XzUSN1 zRvbL%|K`5utJs>w`;*O`mgFv96Zzt%oo3?MuTS?(J6vgZe4cacFUb{mpGC_3jj7#L zEB1Wl&!C)qgZEZT3fG;zc>eFJ^uJU7{d^<4Cj0N5xHmHoZoFgOyv=m#3e(BOS28ST z8HefH>X)tkJhPNFvv=Oc1+l(wUaOohXWe&8_6Ljorv9c`r>#DEhU}l*8k^x(?{&p2 z=cw7Y5@lI-Nw)dZ=e_h~^<6l*Z+3dax#=CX@*%t5zgM{YM^{DH;A)HVGs$CFoD;>2 z&bw5-PsxnG{=@9Yif<2Y6kmU4`GzNcV#<%**Y}H0?ekIh{XIu=9b3|d^P<2P8jubXGMCO*<$eY4xM6WiEc z+S=?gTku@*1yeH*@9(ugXZYE97weABDa$KQAB}pysISFv z16SPVN5UnuJ}>P2@Av;trrx%F{{0^sV{Ht3d7%F(k1C4 zyQ~HNzI2V8Ys>a1x%8^QyS%#e*N^(H9(;W1el@@8`S1BJ7yT6eY`9RcxAfhDKU)eV z_MLkp@n-egGq+-%uM5n0*1FlZX43Yu>!v%*0)O2+x#>jJ#9Bpny{&f1XQUJE75)*O zeYIyVXT!1cMSKi|+zRr#qqgPTcw_0nq+Tc`ZlE)jiW{vKfmd-u7# zz8@D>{b^JBvtB93YvP%fXA@Wi^=)L^oXvN4Zn5Z4`Z)WGlJ|3c*16NNUZ>xA{?Pr} z<2~tTH#51NJHFi7f&a&`6S<$vmQ(~?)>A8;|A+IEXR?3dnPr#w{~BLWvOGP{J)+q3 z<#h$|4dE}+lUGUXW-VH>H>B#;+@AW|eP-_eMZ|;J4o0zc+emWjTNV6WQ1id_b#LaJ z|NoV)t&EfREB)#F{~!0u`c?Z<+xf09x8>2_ed$R4mq+HC_LbBG?p%;v6vxKev zcNUf|pKf>kDgXQaf1Fh-)rwX=bhs7OFVg!h*7Ta!AANP`e=hr@Hh=Z& zcUH%aZ8)c7Se?9l^R%Rg_D6P|%ucS2-}5E=!Kt=N>F+aNceYD)NBrh>mwUCZIkTj8 z;==87xHtR!Q>s~~^lc*7u?-!yOE{{pPc!+Ocbe_b=k|{?%>=H$Kc;s$PtWwB(m7$h zUpyP6SKdrMm6gh`yyx0czs7eT79Br4W%thpllKxu|E`v5oICvO=LyGqTlG%p&aoAF z*uTS!wbJ~}mrwO;pVKfm!0S8%P+ZVGHl{6 z>;9?r$)EqZv^|Uab?lnoDH}=`e%YF5cwR$fA9?v}kLj@N9C-@370WuD^>wwqEfcH2r?9j@Nc`FEFPb&1sP zBo^D>d|U}aFUp-|FD;){{$#iGmbhgP!lMi19{T>!o7S`K)rsPrKIt)P3-Xt5jGt~e zf9=(cZ(ciF8sB;P(cdyq_xL@N198sw@vOS$4C!Ikm!r-7oxM+P?|1y4r}Q@QLeb9I z^0oQrBDc5RdcSt9-5ghWwK(4j`9EB{d+v6u3Y);&{6O+uzsU{n(%P8A=R04ONBBtd zeLnOgUDoX_HBx?I(Nw(nV)*u&?X>P-&Q!r#CCz-GgyKlz@tRYj_~>Gkfb z!CStv9y;w?J~QSgcaQd1-dMi-#m9{oO>aNjmpN|_LzwWj zqK1{rTUY*;z1;Wg#?sCUHqSOkS*wW8I`-|s{Zy{kXPvJWE?e<9XCu3P#O9e>rrwzR zb&}(~KHiUO54qn-`El&=gV*sD*Qb2RYS}Ni_k?*!^3x>g2Gc7S15TLwu4LJH??_er zBpWs6;;$yETs3tNj%}xoOf8k9%r$iz=6|(tNJ5S3j+2 zp5CK9NuQEe&q|uV_?qjI*Y`7Xm&NV9zhvS&^KTnkSH8ErRBAm>@o&`=sZZ~pNAbPn z-to`wWA{qo%>9nP4lmJHzi)MNOWEv7zs>fKe^q_{yi}*U^GURm?eddCt0&kzz4Xm| zfAW_)^{lf_VkZ>tx4sBrN}pLfY1x?KhJ-HCJ|9#9Uj#uS}zI^_)^;hUB;|C^F6)aD@KWqHw^3%HSvd^v9 zuict|YWb#MzUOZyNBFP%ap8Z_-#@0+H#JV_Zl9bmsnV^wCrc}NW8vZr>kpqber(xN zKW*W@ZBJ~u*iL!~*ZpHqyZ?ToyQTW%_l%P7b~9D#+Gsr8r@r(~yq0qEpLc)DO&2)s zde407!-sTPnfx>I+txR{p0%LTtZqq_*caoE?{}X+WqxG3TCLQgu!;J|zJEz{XO@xd z=IiN9T%UaBml50gBHj6horb>{*sq#4{WdAysV^q?>g3!dl`o$EIJ^1J)jP}QOKwq| z`un`n{PwzY|Mxe&H#&8XS9ZsLrblwy<}DYOSp84$-~JhvGDj;bdiGE5;oQ^phcmK+ zuhwb4&6(Y-v)*T1*FSMSwZ_(c;K~ZvOc3UbaDvN4#C8z<=hN z-m5YDn>MuD&))d)!u!A+tK7&+pj@EYRV<@4n?J!&5u^r?RdU8azpPea`1p_cH#n?W&q|=exbh zKHn$(jqiRoZ}}T6d2d4VCzG_@GIGtIPpkCSt$*_U!u#10TAi2EKJTgRiJ$Q~K0|v~ zl|Z>^m-(UaBQq*iXiI-s`#tY;+PbZu5AARMEB?jGjWhX+#rN=u_neMBH1FT?eZs!^ z|Iz~DMdTKqk(j^x{cDN3ulMfWtYEm!A1>eUwWuNMM_c8G3+FjzSJq75V^w2XTD3r0 z`SnY;+w+#pHB6f*rhIbqf4#;3q+D_eh2hs;?Zb7~+~+R5liahn>0{r=g_Y__Av|9aQfyB@ zEhtqltuWCzyytRi#>11&#-I4t{XL&}`Q6bk)kZJApEjwMZs9i&`gL&Oy1$lTrVsW{ zp1=3>2fkzz-o8f9E&9{$ah#Jo{MLN$&snEpjz%=>ovXi?=|@hZ;)G)nFUnulvA_R5 zz0LkqxxlZ#HfNi85B)!WKz(Jqp;YpZNp^W(5B=}n5H5e@zst;hl4s@~uiw2V>R5(- z&B=KSzR#20BK&vbb(06)udL=}%l*20M&j9D-w(XYWA9G9yZ5^#zky6uq2Ed4_j>oV zReRSRzfhKK@GX1A#&|2;snw#F%--+oeWYVCW3x}CE${p7HY+a+JN|yf_}%vMv5r;Q z$=i)~1?5dX{@vcuJP-2Li<-}|BdvK)Vy-L-R4d0z9Ng}n%anVzUyp-^@>}L zy*sn?{Tt7HFP~h0|Nmk6t@KkCrV|U&-(9TTF(-Sj=kfiIVq)_4r{{e4s8)aZ+){e2 zNM@Rs$D0O;GysuPYv}P+oH3 z)$^K#`=xU3^8CHKzBnS?Gob$9;aMuIpQdd-T{HQu@FVxYb4e+E!Pzs@x4ilmd`(Z= zeM*Rh=rzrAX4xG-?Rq1A&HWg>*T&tlbkpUPq4HL&}uGd5q}cX#E6 z{N?kamh;6rx6ZM;t@cNI%TyiH)-BQe3G8#8uW{{jzPTvy_<`Cf<=a?7HZECxFKEp( z@ATKD&#R}Nkm=eSwqH4azS}-y$Nk?|3wmyw_iSF|ng5rUoHu&E&(G|>g+$xFD;4jb z+Z@j@Q&x&uX>H1ueaX4<;wp0OZPnk+7J3x)pV|3 zx+dt?@{9L{Uc5_M`}IlsiR(VIU38nK$tCYxwtu?;+dJibq5t1C7=4M!o>#E+r{rIq z&&HgA#w%Cv{T<6_dgF3?&a?Nbm%i>+RQ2nXRg5`VbDvT4oZoK!S{HVU^sZW~hlWqA zyQ=ROEq&#@uX^vEE#8aWtU}AUf9-Dm!1vvnsm(4X-~CRz%!@ah*mtjRzhm+&qqoXS zWZ!;?y?c&FChrqpc>JH<^Xk`?&(;6RGQC!RWFFAAKtFTY8m`^UnTaL8JxbjAPue`v zyl2j^YuAkTf)zo@@to5SxqeW3@&2&wleMjD7hFFiTcDZBSMYe-J7X)k%du8`Tpupk=HL` z7Fc~A%;H3%SPWX&;wXO)^b-nQy^!{eIGdeToH3!~eUL>#i{WDwQ_poD2Ibz#$!|BX-QICuL-4aK&+C0|pFd39{f)0?@sSt%FIIo<<-Rt1 z>y_8bf4X%pXXmdH+PlH7=)K{W>C@gw{`x&RckRhvyC0mh``q_u?>V8*yhk_BO`j-! z+WXh-fYM*-&1+hA`CU4D(CEtF(AUgC_05G7f6tw~Yw}vVbN83DUcQ#qJmpsWLavO; zwRL@;Qtp~vjNev&{EevbhWuZDJJa4co=w?W$+u;1oO0EF1FLnV>yAXO-=BZ+gS1rp zuexLT`{phzea&uo|E$6N^BHgJlAV2vH-0U*+LHTA`{Bv}ugdDLQ7uWQRyfBU|C&9~ z_w4ycN3XrEGx@n|XXTsF0N(zIZD+sS_5QcNo8^Mw{nP@bkEJp7G8?STzFjWwkN76s zGGBSmowJYqKkSoVvVZEL{rp0|kFo##wD$O`J3-rn>{do+?%RBU;meAU`uu~F+5Wy| zj?`h?-TmcB&4qjCrs`Brj^XisP!_>({y=7dQVn^<-1a(uVrInVQmN0eM!xmG7G1orahpw;Gdpft-UQvc0ONm@P~Qxhm&2Y^Nwj1@4GzX{j$t; z%4;s(4`5zxUoAD`{mRBQg=N`iwqHGvu6jf$~N=A&CiukNQmGA{kgE4zdzI^6C5sR;eR ztIkoY78Y=Q+!wU0{(DZT(j7@F50AHQ_YyyCyK?<1>zc(MTdD>77Tm8_Uo$^f*6ELy z{(aXq^AFyb#IEAMIB&Ae@0b;;dze2Xa2S4M! zz?3(^#{TUGSw`o(W{L54%YxVckGz!8{yuX-?6!UFMN|LXdzYCP|Hu38w%?_<>z`KD zKl&BD?(@VgSN{toe)>`8U{-v4$+p9F2{AQ}8#wPf&fMZ0=N`vbKDkVF6VJ^nr~Rwf z9G)m?aAo!Ltdyy@%V%d_b6fI{uU=M1q0a33G{L$XZ>R5D_oaeY`K`<6y*F*5zyE%J z&vr_-{iCPl6J#Iq{My*}Lr~6sN3rITo4Y&oOP+hbEdS@vdiNgR2jRrOwfpp*>=pbb zbp7)WGaseBCl>SGdDXUd--jK?U%%UHxbn5Z_20WQud!JiYyIpzZ>zvv>tnv=Z)ZCF zC@a6G`^A56Qw*!s`=3YVyT3Avf3mf=F3LNB_qx~Ka-SEAO>Z=RvdG@yvh#Vi_~)~# z&n{-YJEm*@q~Tih#ruv1_sz}*2lL-KeDm0q#J{35qF?79m=mEIlkT%tWd?(|ywba= zitW`g-0~(K{e?}(bwBh~?_Z$4skeaRr=G#|Fueu$o%TO{%vL2adHJr#dOw_={atX~ zHmYLAj$)=w(zPb{lP~{UVtVm!+N_^>v%Wn&t+Ht1^PAl9rtzY>j`O{9&S~v;=6rrm z=>6}>hF|voe;fwt;<@6XeOWbGe;=^1sXbpA{&+_?_->Wbut9XQzZ*xc}eh`+?U}PhRl8^L~BX z6JgCMqC!*79~C;j^1w9FQ(Ebw>HCs+*PgiWBLDX6t?wK8o}Dm0^RfHfZ?-=cm44@U zZtmWCv9)?vglx={^d};x7)q?2)x4UP^A|hT?=LZ`t7*G>?~iN~&$q=quLWOpoWBy- z=Cq9~_&LvkW!1%14eb1GeJ=l{i+R77nxvX7^Y^*5P2{yk=5oFCb^TTiQqK$bZFwAX zp|3dI`hV<+;H9$ZjFZ=0^A?a^azFQVOWm@WA4)#Q1$kRPsJ$Om-@Tyii|Z2m)K9No z2kOU%>`lFwe(8L==&G;xzWTnaey8=#-1ys!_k2sL@5`U~cJI5N;eB}xKT`s4Wm+xuEXpQUH~4wD_*wrf0^5|5+$4X|7$f5t=;opd`7-$MD?8X z()lmktuOq&R&TtzH}iMWUH0ehH&S9d*YCbjDQPw_Zi$)V`_|S4`*{VF&Trk&zWKS* zrDc}0-&AhN-`lqN*h#hcb-Uyvk5$(1suauSNafw6KgDs`otH%twX2r&tliLEwc59$ z^BU*S2YR=z;o*k%8A!JyTdx9du-*1ByL@q1FG{lZ~~ZRMByJSQ&S zPuLW8+Tp~OWv3sW(EDM(c#Yl-8`-$ClHn1;vwG+FADwcu=W~hHbv9*oMfQSxsqdwa z_i)cS3D(7UR-i!vTc9PQOodp>z8IvcE9_*Y!%D9 zroT&WPC9(&$%^;~(ZOzN|KgXOYS(^X@j18EXDdsd@*UnU=d=Bs<+Nwo{XH}9-)x@r z!&?hBSElI&pI=?ME>F`&b>@}1o$KeVjB8og2OC`c(E# zD3Dts$Tz#Z>z&EtdoydAJ+s^UN|o+4R>*(ibrhR$Z&rZR_Q znNu$I{4Ubm-(Z=M^z5Tb<~qOSYnNZ1UVUL>^@a797h2aStnHj0E%s~MwbP61^Vimg zm+{v)%=dRczh#empIS`XJ@?)2#{|!9PT%C_XRb1N-R&^0xtiMYGg);ttz%#A-`GKPAt7{n_&0seR8-El-!^ZJv*riS5`QK=D$E}|MNUI+%~#bwbw88?o@$iXSh_aalg1*_RDtaFYgWF zi!WRk{%~|#@VSs%+hUsnj%}TG?2c96)|_=eX?!(C z|G}<(`*p8xy88Cbof0L~E{(n^e^1Ci4d*?UPxlyGvgERS!_NA~^ zdkOunNxwGBXutTCeNWG>S<3n1jq?qczvcZOR3+O3U!|vaYi`$myRLL!(*FE)LLaBU zFV9-T`lI*v+4E_O!n$r;3#&Qd`y+emx!-U8pM3Twy8V4&)8?bM-oM!>w@~+_&>?fL zOZDZD-e2uRk_xy|7Nbb=+u~ z{iJoLWxL~NKebu>rCiW!gSPia&Hc&w*FPHkmF~#DB6@$*xtj}4%?-UFu>Y&<_7M1-@5MF(@W}qb^q|6Id9eV;s30a3-1I! zt2uE?D)Zg{#-B9@cLvXWUE}jl|EkdQu+pucSwAkHdD!%o^(!uG`3e6Fb+%l#FpfPR zG5>hvBk@N{x{k5KJqwjlK=BviP>DRWjblj&i@w-ag}g```bazLP%e z|8W1>|JMIs?Dzls+P0&LB@OX3&RIW69MsXlnm-k7BM zsq@q0Q{t22)9)wzzZ>_s|5y0$?LWS=Ok&>Ic*x=Xffok?8!s~%GjC>|&a#e6hs#E^ zLVtr|V&nzBw&s-wza(r=SpQ(dgT#ZM9{hMvcrfyyc9S`aIGa4L9=DzBpYRX8526p6 z9`tXDXWP&Jj{oMu01e$MT2~COs9v$XqJ72O#KOcVM9#{_M71R5#hxYFlh%1h%_yn9 z6zf^8es}tpfJ@1q=hgmdzudZH`=opRzf^MCCm+sy81^vk;lhU#AAb5^_%QQf@{Uld~sgoj5)*S5i|lQ_|O%%cw0~ zX11Yvvj66HH`CM3pF4S0^GyCM`Lp&$^+o?!e{yq)iHC@q>7LP-35$rU@Oil8(8<H1}7ZkM_H z>Di=eudpe5;$CUpC{Y+=)l3U@jEA&>qi@UphSN+!ouNGc? zyh=Uv{Mvo{etpYYUKXC~dw0#Qo2AdM#|D%K-kq~+meui`E+_d=O^v;9AP19M|^V}2J zWBNnl*Qb~Q*Miayv0t8cKe&EyzEju!@^|jn?`Q6&y?!Hi8y~Up| zzHeVl_LrxAU!MN^;QY}2kbm3%&V75D_N{S^w{wkOGlenp`|`EMzmz7rd`S&SvH4Y$ z^MvKg(_JT=zC3+-g8Rkk?1|fzUMtOaD_fvHv1d)sqMnJ8myO=#hfT4ys}AdXBJ#w$ zF09V;-^HK7pY^pSH2vmYYOs34ih`7f75?#k`g6j6#5`PesGobk&YjR5+daC{M9=C) zMU>QE+?Av$7h1KRf)J*UkRatuIa=P)(lJ6K|AyveWyV(J!SJ-gTv~U(I`Q`oJs2tA|(juiLlc z*REI5tFr4B>wk`^E!2kKK=656YG_Jny}==9bQWwQp^7`1ho{drQl&gk8B4yLrd6b+fFq#jf57EUrGAYr6f~otWa-vs+EY zuf+ru*PVSct2q1IO3T&H&di!=y8Noo>X}i-OP}4C#cAq(t!K67ddXnpU1=d_6HS$~ z+OG4glZ?2$ep$8e_F3Pqo?YF&n)`*bUujusSn1_g9;I2OUZpFn7FuoCzwOoGRbNBp zL)Na2ivAjRb$96dkiAh~*RM)nEw`#_!>fO*j)so#rMw5+H;HT3DeQ+uaQ zom#C`9{O(0oxq)IKTSW?b!zR@Xf5;be=$FUpY~6U4_|kq^m89~*_{2gufLt$&rC>H znf2_~X4muX?cbU8dHw9~IrsMd+iP0ed)=QOAIYEpQ+TD{ zwq9dir0M3OXYXcOr{~Sznfc7zc>lTE^TVfpuf6xYa{klYGrvvVPu|o2`Px(6)72X9 z@9mjqdARsj>i)@lt*S%6&AyqRvVY!RuWzT;gx~$E^-urbd;foz_oq&t5`4GgYUG!D zD$mwSW!b-|Yx$S^rZefE(DYill5g`~Tq$Yww)qmA{`v0mOBR-$waxDKhVdMm{=c(* z`fI_@pr#w{2drY&t*GArKP_VId=2MUUvvdpWH+!|7_v@^{j+xBPQ8C-C!ECVqf-pO zsP*uRORy~FPHB&1 zrDTj_h5aY5gnfUuo|xSboaV6oV_3x$gXK3DZERFhJf@qn`?L5W zU&Wda47FT_X$wsbebGG<(Hgb0SlUV5NtSbd|B`nC%ikS{QAw6`GH0Hyw9?vTF5@wU zE7w&IM6_JEcp>4?i=vK=yuB(lx|25YX3Fl8tK8wuu$RxbNwY+}wC(%jxiJ=~6q{lNIo8AJmNrl{t=X0)kYw%bs?T(Pwgz#M|0r4|_av1Y-zKHmd zHl5{K^$)Y@EoC0Z49@IO-Qs@l^`r*9mdN~v>>P`Q6OZZtF-R;?`opo|oX~~t50f{{ z+3=3NSfH-cz)G<0rYd7B$9|@s47sN)!g%OR zq>7-#cb99PDNDSS%#;rOaF_V+qPvlI;$qHzSHBX^lv>w}?3OasBHcrKTsBm9TD_H; z?sCnWCl|bMmJ6yNwh0Go^((m(QU&m|_D7Q>wp%~(9QB`7 z;~0;htv}brYAiY1`i9l>MTugqj!yaYGR3$0DYdGG%UUPLOp>^K=%ilNTyeir&!isi zOm{^yBTiY}l=E6OZ6EAZJ_nw-p>CYl`l9+_d&|bDt-ExdDN4p_9jOz~Q(yD?4r@>P zr;;aeE0k+Cy$%!EcwFtVzlh;H*L2~uxgv(^T))`P{M7ZLSn#!ZrTV0O;tTChP6=$W z6M0*ZJE6{H#f-m-KGj_}?{{xBPW*O1=~m#H>#D_JXCy=xUr&6cmG-84{>1!eUu+_=uQ98`#W2<@95WY-RhfiZ}FP-8A92jdwUi& z2ToCHt#kj}@YkT#)}(80SCVk>#l0H8J*MpRbTwb}I5udfeN~UQ*559<=aahp1ycVk zzOgr?#P(E;du0B~7ZqRFre8_(_%>Uk+D&VhKEzfEAhSm;y$5kyc1M2%D(FUjVTW9z3F zd3$qacgpH7{Zn#m&6Yc=b1Pn2EcjQFlk@VT-pecTU2C;c*Ic^U>>YV}=@%#S$+2Z3)=^Q;KUn{AhAi-Xxn;ZW1y$MkOMUlwTfJU! zmSOsr?XD4rE=~LCdHmY;RJ&Vc3O@I5Us|^9#_y7ntvx%n7r!gX36y*~ZEx3Ir|{Df zpN+m~9I{P);V$`2v&sI_zM0-FwdtJSGB`gAB+AD+&7W?-CsHPjK#cA@SVHXCGIdaW>9LD?PAO^6iRc>#you z(5aofbHy>Xw2A9ynT2k?w)sVNY5jDaTT#B&t24vJ%tJ0O%es~zmc2kKI{NIK)P?O= z&xD%ho;&m}H2MCjjk=LM6{fpl&fc4QbYD#Gzp%%jQy#7kHCDg6MV0&ihO+72VqdRc z+mT|ZR=PQ6(+&OCX}eRtT7EeDWzD{O_iweXse5<-_?48e-}kNMi#(GmTO6|dw9JcI zXOHXMvISYOo5H$Mc^RK6y}q9`bH7zX?a3-<9iNF?`D9Q1iqg50|C;&N^uL0eOcR_p zWuK83-Db9c-}3ZCcjhuH^M*a*AMO{Op2#nuQZDl;F=750+5SK55BQ(SHO6rL$ltlE zXFAWMYNb!93AIoC-1li-bN`XPbGfo`gRTAd!W4-Qn~UZmA6@VB2T80uWInDh39VOT;DnA__LS;Y*9y^U5r_DiRp2e($Y&_QOQiVyk>5h z{OyvXm6!TUo!BL-OD4a&{*ywL5pomjPdxrK!$#5nRPd&dX(x+*F1OJ>f1>zPla0Ri6V=MdFtz(C&rf(7 zIs19}eze+Pb$^G=ulkCa)KKuNio#riSy`ypcr0=GHylJdOyEkZ` z$SvA_N4Yz7`!2V;+UcuzUr8$69<$_K@b?$ruTJy4X>?Lz@0zneI#0XJwa=XC!k(hb zy=Cr={7Ig(tyg9oHqu^P8@4+&EHdz7y6iO7Epv@FuE=mH`LX44x0=}8lBDx@9zF8h zd35qE-Qd*4{WC0dtXA&0Yqa~#mNkn~nUDMH#FQn=;EqR#kd`|LHR*9d19 z*Uy|HtsD0&-@5SThtKLWxhJay_V3rRd3vMlG*jy7+0oZ8tXh=4%;#jRn#kmUZF^ch zr6T4}i4gPDe|2@?+&u}F;iq1tc0N|u+rEZ%@-+VIqI%wo#eMq}4Dy5Qd}kO1PW#!_ zrn~tzU#ysh)VD3`H)mxl)hRV<=}%g)zUQ-2r|aRkDJ|ZzA0IjDsg-?DQdBx9q(8|; z<;tCI6=Ba)#md(7IdVvy+nW>&3nuX;ZGb&wlmCSL)e$73a%)eq|Uhnz?i13zfpC*X5d5??)`|Dvdo< zpBvd&8yV|9bIQgOizD2Zf4D!hXr0JvZrg4HMZY-~g^!MO3abw|<9~Z#kWSLp!17LA z&P#PWRb_wK&3$nC*V50Q8vb|h^S|A-KxcdDrg(MxxdJEFz5Kmp>E+!^#FtMm^XwOk zKIpqTeD&@uf4wka%`mSurb(T@^Bc~5J~WLbwORaHz53om+_x`Xl6ufuE3`@P+o8IX zTf?dzHn6PCjk5^MQH`_kytPU*&O+z#TOa*7O1f{{_Y|~E`RTRd=qJ^H{Gtba|F36l z{c9TI+kE`bADnHCR*rm>^!_qm-(h33*D4mrPN@kxHO#pC_U7EXH(#E#Da(~t z+TkLz|-MWCC7i=JmcMyciZMwb!37vi$&b#W^>0Fd9{ji zA0dW&ZCm-X{e*dZ>`$nxgeXmTV7IXGLUxmn?B8cQP5PR};;&^)m8$3uUbd}w(>7l= zH}ln{dtY{Zx+Sq@ty$PiyT8dUR$oVxKta*9R##=#La+h@H$vQ{x?+R>M3Yo}EO>v!!*EERYf+7a3x3**6!BbK%Chxe_%N}uFnEU;PHr?g#IF;vRUKEQg%72iZ;@9zU zUqySz)x#O#Ro@vFoG;E~ajYo!NfFY& z^1ITUr>e^Q#Y_#;tHFufk?EiQ#GbjgQ#qp9^wejS7dKlCG{%zOVn1wee3Q8M8EPqb9k}-YSW6O<5yO{GTm~!qrms=D(Me7 zNn&+IS8}(mj0=1HO7yGMzP-&qxBt-R)_dnrVi;ul-f?n#qs8XeK}V8*d1f6eKWfeN zyz#}V!kKJEoDn|{T@Cv2NAc=}9lN^js3yI1H-FW!Z|3KlZs!ZzHhI^teN-#CQ9azr ze^r})cJo%gbD?XF&AB7H>(TaQ+M)6%w}#4uu1m0+jLcBZI*py+n{~v1r}evC>$J;9ANZKvZ8>1Fbb@Pr#w})xZIh&= zZfcxU+P?lxn`THt^PhX_^Oo-Zw^4n*!~Vhz%xW`iERUJyu^?Zyf>C|9zT$jIBM}MC|5;o>eQCL}kod64maNa1J3;FF2yvh`SL*p`Nfzag>k@b zKw^3-mwrHEQEFa^f}w(eLJ*gJP-=00X;E@&v4V|_en3%va&T%%u|l+dfQy@gen@IX ziGsdoZen_>6O?v_(w?zg`mPlv?!hIAC8;32?gk1*ARB|Y^xX{=Kqu4&aoO2HPSFL) zyBR1LnLzAxGgL4F#UkiLT_XjUps|9H8Cc%V4y@5PH6=6ADZfG?+CafT!OYM|!Q9d) zmP_9`Kd&S;ucTN3VuYO?$mgI00n!bk;U^T+$-hMT(g+?RAms%8W)2I9)VvgspDYlG zKBTfBRY8BS_>I`evPAQcAt+KoJ{xqgMXZkuAvp+<{tQ9Me^7c!-_1yY!~|#vPp{Av zXb8%ugU5G7#}axe0g5#YZxn-ymD0Qt1#>Qa@6429g=iy?x*!Emlm;n)N|qo6P#z3Y z04Lla1#n~sDS$&I2>EbskbO||@l{WTrUsU#C>7Myy{}K{?d((gQ6KMm&!Jw_?zRC> zNcq)u_b=zX;Lg}3#a6Xo^?P}@{x=;PmQL8`Bc{?lwJP_gz$U)_B_EEv*8dA$@PCWo z{~z!Fzs~>vJO9UYDW(61>;Lrc|GVA(pZ%Z5{@Omf|9zS-R{4Xu{@?5Ue+~bClK=lX z|Noc#>+=6U|G%K|Q0r^I+^av_y;` zU)cXh&$fSl$g6`zJ6U{r?e`uj*r2n0=S8u;IempsE=lJuToCxDWY+4ox-+3HXLr;+ zn%ca(b@BfxF`Lc9_dN3F&a8Od)wpfvl^1=&r~G?XR$W(jo8^7!#7?2Jp{EOudv4M5 z{5+r4cb)y?R~K*3kLY+_G`IZm`$=hko_yB0aw+?l=H8U@NZmg@H)dJ;EnZ(TrTMAG zy5cvXy_1iG_BKcK6;=o-#xD5y)3foK>W_)hGhSrNuM~Z+?c2F7OZs`~zjV*b^;shG zr~fDss(su1ZnJgx!kx!{YHH8pKFb>TMy7hz$K~-`cl{22sdMWm=QNFssGpt>|FCrb zm=e6dLv4qW*4ICiXaAmhsjIGv=j82qHLddl&g%zwt*E*9jYI0BAD@Z%$2YpRm#=c@ z&i!NJ>3rK+xGKKL@8$AKDfcEmlXvN|(g$#UAAFK+J-{G7OIr`umkd+Ew&Pfyt|b@^_rXK{Yw-Sac#_R9<1yKC(q zbC_at&cN5Ai$y!z=TkC(^&ttoo4i|uTvUigvls=v+x>z~|L<(<1y z!o4f)(bMH$OZq;qZ&JIkOQkoWTJ`9LOPk+(lF3~6uxqNyr6$i>i6G_o^*?^7)sD=jA#NLD_e^Ji0v`U!47C?YrBhS6<<{KnX}Q=hecPPjSf@(tgR_}a#VW!!^ew%KN$=km^c&nH* zfAz*CwV6kz_N#2`S`f(QnQ8C)K_`Buhu4RzlTRDTYHC*f6WbJ9JU3b+S3@Se_w4UU z%0BlpH_r%Hp132@?DX!EmH#hKY+GHidvd}@Gey%SS^dH5*H)P;U#t$(7dxikYt-M7 zswueSpMQU-jOFe>2W(zBNtk&ufA;X;O_iId8Me9laj^2{v{?cA_Mw-y+&;4@^M&H1 z#~GhH_FXFT(n#%^Xm_P~LV)va_r=Q}o}K#gSBucOTCwL-Jx`r3oD$1?Tm9OLw~Dvz zz6ezZ>8MAAYyFE+5p;U}FXlq?BVT>iru&Xu_1Sr=ZNJU34d4IP z^mF$464O&A$8yRFOkcQu+_rOqT=6OORFw-yJFaVnFR9TIFMMq0r8?oIvRj}r>xXEY z%mAe~hst!K_x;UD<&?_pycqKLO7)bZPTRGQJ@ZoEw&|6}yb#SV_jrY}3hB^uTH zvT$`FuVUyDOD1jG((heYIk-1hU0;6bDu;Tf*`|zTHK*q4Gq2)REMH>S6fD0_XN73l zs?|1E-M*bFJ;n5sC1l2ntvkIGZ~L#@A@8T!n|5>hvIsTSc{y7@6_)4he*P=yXYB6w zlt~}2w0BR*HH$EllDx9O+r8?G;koI7#?8v-EJ|Lc6+BrQc*-nxS7uMn;}btFCI86z zF5s&$S(5X-q-Qa+v%vztSqiO;^h0dH}2hZ z@z3TZj%(QNE$(%@l{hnJ)}_9s+P^u!<#isuJuf1!`RL7gzYg|u@zj1$?w!?V5n9}@ z?Zlj>oITrnUH83($F6nkwKwQvidOy>qp7gqa?D0Ywj=($%YOv6Nxalp%vP^wRW8JR z^YBJY={)VRSe=X^n;cwrk-Ld-Mv5ER7LbhS1 z;or@c^G!Ry^3(*AE!ItLz09mL9$T8(=f62&x7H`hXVT*n45x&xH|3v~wk|r>cvmMg z>ySoP#PF7}%u(mQRncDThf(V`C#>vkkARqKD= zB^}mO{c37n1fSE(U@?)8JX>a%^WEK$x%Q^MQRn6P8g^x#PP=l}T#c{Vces)7^iw|e zTibVrC8h5O5?2e6es}qQl)LnCEhbkP?^(}o8p)x7a^oGd8 z23vdGX4x5JODx-VQDcYnOsU4F7f<(hJwMQrRGIRbXHJye^`%$JetdPdzNTtEKiPEB zGwrb8V>`<%{g}@;xgBd=usd$yPMgCrD<9i%XKr#RpS!Go=ei2pXA>>M_sD%dF{k_9 zg85?3CC3$cm-=s3i;6F2`JlP;_L+cfnb&Tdt~z9?-@An;dFNqozSF9gqp$WCF5=>D zoL|(lc#Y|Uc{e<^FK;qDJ;5S<`fXmv;$tt(S~`?9OI|HmIiQ})#! zw6aodG7@BZke4LG8SEV6(X;qUeINH+8|^x!YjXXyVO7;k_D$(4{~xyU+P9>+^vW{V z^v}#WSGMq+{p{rs(pC7Ra^gSl;-il~IZtY}d-AOB^7|b(-4-X=mEXx(JaLv$l3nD7 zH3o%S+MC>xADxI3ws}2+U!mt+a^r>zJQr`?KAgC)q^>C|vS&rkJ@Jke7gi|mIy}Yt zQu==84z5&BrmpENuq(Lr!?EPkr}IqPiap;Ma9D5)b=s=TUdrdFyfKGE zV%f8-?3~RpM{U=hVsun|yp&NuaE^^>=+B(+OKy_{bfhfU*JKp?uU>Y1*S4Q0`%820 z$yF(ZU%9ZD(SDx+%Z@y^PcriqYui?5g|sYLoe(K$zoo=;@fvv+!`IWD{54pjb=$=g35TVxoV?by$75@*Ose9+Uy|(hr<}YT&n|7Y zDlh9j{23(cxIbfCuY6{%O~$rdy;Rq`H%-sQygj$>=Ces1+w0P=TV%Jl9^o)fpD(!K z;hGt_dNW0tW7B6bsGXO&GE=Q#r#;83g0LQApBLKN$M)G7Uc2pcVj4$B;fa+ZFSvJD zo_l;VBjiqdS!LGcXKy=yUYv3^mMyPOrMhrB-<>-f>!;hWPl`N!ts;HW#Wiey|I4_$ zu8P{9%Q8E8f$H+LiN$}Gh+4Lre#%`x^ShGb>@%{d$NKhITPeF6RxUp|QQ41m)Bf`= z`H^BKR=Y};%ijECk{T6kpSo_7exmf!&(pHr$YxG{eralDXM~aSk2!BwHax%jEYIIL z?NY%`$0sZ11_dmB)$w?x`|L@Y55v9$A3AyWq`1-KmX^teybVUn<#InwW<0eLx>>M-hkjHnV$k}kW@!M^EI@FdBPsqs&{nBUq~ zOrOwfX}i-)ki9tV%aNKZmajcND9eBCib!~U@+teq1JAnUex1r(cUm;D<44Qt-BJ-t zi`P$bo5#9fQqo(i#V?{flfJn}{r0+3)YrZ5tEkna_q&=qehban_c0~hVu!~|D-M+@ zubq4}YI7@))9^-7R*vgUN+H zvq}64*`5s%drVI8PvpAr?(Wi)tuFdo_T}nT9_B95aQw{Z;9EQAPe}b@C)^oQF*^HLtUiRK1cH4tLd)@S;J$O zt;)#l;lY47fBn7OI4PnmgnFkko1 zkSO&Ok#NVZ>#w?Ej_6Tzii^LD4upJ_cGwmxr4?^N6>-N|+TaHQa!%SqpZ-yN1^cT-p{ zlxL##Rk-1CLEgfW>8dOelXtg%srL}O_Mq0XCq~R9OV37l<^0<^#~o+5Tv)O7u;l?q z``>28SEVicN@Ns0ZtMT|IR5ra!8vZnTZKDcpAw%V*W_m9&vdACV`cRWjjpw%;X;Q*DaRzfm`MZhijjKB<{sK5AsBRB+TOTTVasYR~8IZH{bYu=a%fL4u#(YRNarnoT|K% zc=PzXcjp~;P7Qt=JhMT(!Y;EUM0RONOV`Q?2Q>00wQp}P;Q4yv&&~%DQah`%*E4h$ z|EOF%vH0TlGj+BLg$v< zIW}=$8+*{7OEqa;n|7y3yWisE40uqim-_zMhC8Nw=UKlU>(H}T)h|DL>^%FM9o)0l z3(fjybt6#YMcwiDveS;s?k|t1i?O;LbL!5HSgUVh`>w`sE1a^kaCga@x0SIVai0ji zDZMV0wYSbJ-D3WeRV3fU=uKb557zd+z0LW&lIb!}Ji7|7th!-%<$S+)V;`gXj&rAV z-el`7(98(2J!f7yKQB}w*D_fMKp zS0Wiob>@|E799Kf(8MFXI-p#IB|NDp^MX={n5Fons?FLVr*|~}{ni}BHp%7u%qf~- zPkpSWf7sj@W>9G3T<4con zUhm?*9ev{Ev`KDkoC|grN&3xBcN5iP3+romkmE9`#Tdp8^4pWpFN+Yy;?xYdDW#8 zOP{>bm+s}-vY;)+E^p75xaBj#zl#6fvE$gYkpJ>hoctB5Puz%q5bS#Fo>SY;B1xw7 zYON%zy*o_a2L6Am`=oh$xKBx37$o1_-tAtf=pZ$AMFunB7 zu|UfS{aN{+4nK|mwbM3evRlPll|VL;DJM&IE@yr5mSw>$d2#Xj!uPDx>t0^`d&O9* zxaCdVE1tbiMRe2itwO$iR-8I%!t124j}z*y9F@8L^JdMpqbVUP{@yH@pjz^Mq7I{1 zW@)^PNbXG4|8MgbE!pJt&)R1B?vVYpaXtHdpTAxvui|L>6 z__Jo&x8L)*<)8F4f4VPeRJM5HE47+m3+H)`FLZ#t?{wSUcX&v$3>6u>b=}1H=grNuN5%4C13w?=aXB%WAh4Td;fg%o{9I{ zE{Ti~zDXbS_a%6Q|9x<8x^lggx2Kxnf}7noJ;BRdUHKLqcV73pZrYv$uWzzgF9>_d zvVOmbtLw{0x+~5!nC-h(ti-9SqP?YP-Y2t;rr660cYF+HZ%i!7cp<}8n;94L(f)6= zL;SHH$L;?mzl#i#c&t776L;#%pMUE!|Nr?{_qB9S{d&VwzrM?FIO;bmgXj3e6%vUt zSLbf~8_*iGe!kVA58tMW>~CFnUd=~g(kGcZ>2?=C!N)IFtx0YznY+{D=spSk-1&-H z>(ts_=0Dcx_!$vUmpv|mri`?7TM{rCs+*EgqqHttAo zTYc+!-=15XvD?41CmvGF*tcrau{n#>89eO%m^dzGoM0k;{ol6t{$8GEmpU~pS$RgK zF#YY4RcG#`{N5wPx$UOa#fLk;T(vv$u>H!+s-59HQ=`*=Dy7f99r3h#POh5oQZ6aY z^ocRn@yfwxm~KfN_qgY&u)S;=_ojR;rOu?{kOR6sp?W9o$ws_WoXg}Gd1%R@B7NuT zVgYxV{SzkCs<{UW9@=E3bSs|sQ{v@(laePzGFv}P3s;T%&9msSL=G6{ciP`15ihf?N}=?yyH&&A$; zqwM-YKki~}j^Llg>%YwLo+`6SWyWL6=}qpJc70k|v!X-h<&Kr+;R~j@y`E&U>Z(i7 zRGcl)$6pt6NAJR|_TM`e%CTszqpc3^ zo9~hSTccsqoIZ_oygZQ-Z;> zawRuOa`B}s(T#8HU6%bf?EBdY!_}X{t$zfv)yS(9)Gd?@?Opt=$!z|ygZ+M6YPMub z>$|mmzZ55X29;)IJ9)zyPJ0|{`}~OUiE2t#6E_+Eev|w7#JZgIVmWoq45xknEUtYX*qZi8es1a09Uo?VvttfW zw|lDZT4}KDqFtAZ*2Nt2jTbd|r+;BOW@e>xdAIp(UyH|IYJS(}%&Xr2*Zy1Jgc}nc zXPC~|W^nw?$1|%o$+)L-Jq-=HvM)(xZ@8-b@6S#@XGESn{*c{Dklo?!-IKNvUsH$j^GTT<;P|T7`H|GY1$y|NN zzTLfN@#C1xtyAxpKHwC9zK@5r_QQ+-F5%;!v!8LDvpZ<|0iCxJCU|0 z@AKuG6{1G7(@yfHgQU+NwcAnmZu6SO>}RJWI-dL2vn|2V?0my3|A+vy#}nQ(Cr3=! ztCu0aIc}#h-=C7toM!vI9ll-{&N)AI-EYoqhGs7-t#TJ|&g=jAFO_xh-q@umVfNtVfl7tHg{tR z|KYeDgE&F^=X;&%s(NQRHsXy`m;eR z>p{U)yJ|g?mbngf>2n-@EC0CId5$&I*{?1o{>qnwn~M&%-Y!jhr&uNG>bW?kpm*N3 zCv{O7oVREFW?7(|BXBrl%D)-1Gp@AfS^vH@z3++j?=IdKm2+y3x;?Tvw=ZdtbX@v< zw`8l|+jjb-$8Wr7)xY`t!pb?d=VJOd=Sv?i`6ep8@sq{xoVvL5_>B|Q`ggtkF!@zb zqikJKoYn?o@mhA{kiEU119Pg>_(Y^7S<>a(S9SOHtt>bYTCVlumqq@8#GPMfnr}F} z~mbCz6_OB>psB!};>jc4GPwLs!I&u!k2n;9ZEp0lmWF_G_U;|wW$<}~B+f`Dzg zJHoOVszq4T4_)1~{qzd^X`d~g2Q9Pr`6jma_T&>vGs?EqMy&`GJD2)W>sPJik-00m za-I~_P0^b8KZ@^T@3X~>#mPQBhmH1@Z{GIz*ydFe<->EUpKBc3nswXyHB(P}_9bhZ z$yz3{kG{otte>e@_*?nx!Zr8&&RHKqmMOqk7 zykD0*p;ef*x6kiLXw89tw;I3C)SZ$0GvtNrjLtvb%ocn6^LetpUzU|It!VFFd(~L& zo%ihYzL|!6oO!Be{++N*`}f+n-s{q1nQQXybkjl$m8~ZNuefXwmHd!&T6D=v5xo=d zWy17V&)#_WoX6=u<$T#&Qvcol!*IgbLiW_N_C}jKNjK(kJItGNWU+2;<`t9uEpGx{ zKj~k}3;w3ZZ}c-++{jt6GNb;Krg2E?rkL=kskO^QkFK?_J$+Mtm+j2POuqA5FYPYb z*JyS7f7I7+ISU)RGBr6|BiD7`Q~l_>BDsC}D)Vsu)nA@0si=GX^P=tAMOlB7TlfF4 z^VxDMP4q@46YIjG%`VT+Da$(YEG~M!J~V&-j0967yD*ayX?ID!%&QtVeoXB2IuSMF zSZm~Qfpz`f7c!38uI)>gwhuDoO)Rz9ynVvriI*pAocM}S@KCYdTvG-Wdk2sSeY8fM;G{xrWZVwL#u&zHCEB1zi&uyQ{ zsdI~78z=fnTd#77jQyE#dz0O)!q`b}t9I~Ae|7Qk%!ffS!Bx`kkJD1XXx zQM73M>MM=4FN{Sh>rQkf3;o$x>3;5@m)_;qZ+sPn7xIS76l|C@`(NH;kIl=rU6r^o zliOm&%HWyl=Omf$beyqdcR0Ly!irT%MUlB>bN@E3Jh*oM8~-L>-gO1J-8?2Xeu_IT z&DL2Xe{fUK_K9H^SWe4d^AIzUViSE5qQ$}eefAu$b*~(~ELUsw3uZO{ZQL^Xu|{&r z!Ig7DA2xlR?6&k1bHX{-mvfj|FZkIAYc1G$iOWnV@-fdxpZ^jP>QjS0?)z-M`as%` zvsxRz?>I2w?a^A53vYHDDLeC3J5RfIZ~XopUimYw&o7y}&m(cE$lc0``n`PL?o>`Z z+VYukyXj}8Zq7~Iyp1wq&1xTdW0xka+uw5g6^GT)+Oze%+n3h=Z@k^aRL1%GP(4)+k8EL zRmhZxJAV_8zb#ob`!~;w2+{QoUm}&+CtoZ)7xHGk{mqM4CqGzkzu)2QG~PRZuHKZH zWV89_>e`us@t!rc@>i#Nb-qhK{ByeU{M+hx8&rbVz1sIOziZiE(+{`w`nS&f9(Z%^ ze`(*#k5}l4_dmbAXX#d(dHJ*3*0svGGaosnC6*?yv3S4V)v(H-hZp3R+bv#p=&a`5 z(8bzw{@rmoR>%D6#OltTTTaK7e;>W2FBd7y+WD{esX~Bi%DW!H^a`m!30I z{_Ng%;ofU^sykk37tJ~UN?X`1Y5nRuSqo&lQsJDEf z6MpFG9%0w?opF=OXIg$t*2*vIVsUyU%$TU&eqev(k@m$!j`~TDv59Z>?)DZ4v*s&-tX}o$8zS z%0qJIfBnCT_j6ylos+5J<&;@#o_+ncD9LNJ^ekiXDRYV?mQH8P)Av{s)fVv4bJ6Fp z&s~$kg&&{2b-MJP2y1L>+G$a%IcHy9U^rYE!R5!5EByY3+hUW+d!o+en|!i$dF!Z` znsvT4>docQ9c#UJs^*_PE6Lt|G-~GF^P8XDF8#3hvu|eXnyELZx0?!IJN4$X)7kBZ zqHZ4B%=3Ah^pw|US{Jds{IjFPvu9%wixZ>M+MzIT`>!>WBp?!k70 zd=Zmf3DvQDuD@R=xvX3e=;gX(rv*e&lT&Cyn6AS_h;^v?)_wSZ|2@@J`5rz-)HT4-@S0b#@*iqmflZ0obg+SOM#(C z<|XHmyBXI%oMYO%Q12aQ+I-i>yxaQUrFBkZoxA(DEaP{F+Rjfib)L;X<(T|?&js7? zPm8Oa|8mx?dgeYi{PzQQ(cE^XJLmEv`;0GXRi~df_1-_XtMAiVMm3$xkeg*^Oxw1f z7rm7$@_wa{pX*!mYYp3Lr{(!wYN$Sc*SecaV`I(ksqZINq)lJ7xvWiSzKNCP)Nkrn zKU@EO@z+GU&eF#G*8G}N8kNtpcmG{_&hB!x;Au_ay0zig_s{dapBdqMNB@_~SAil}2J@>tIxPjdH9n@yB8pF1vu_@oXyWkRIBn^yh1&#fuUZ&#R)T-m`q;+&-IDt` zCD*R>m%3ZW^mftVlba{8Wr)vh-fe#;Qa7Z${z7cUt>kYOFZP+ewKD!8exWgrLgIxTL?Psb4D;`0{i>l*%*G|7ja zU-fs3-1)zO3ZHfz;`mlRtH8YNccr3aX;YAy)YG^X+|%CZ%oUloQhM)UjfRRyh0OHw zTcy#fzr8a_vi$zgQ!LMB?%j5V9|b4-jH)hHL|t0(>`Lg`n5nmAxN~ylS(21vPw=Fk zUTj{*iJ-`cj@-f@ac>zQ#<)Kt!JI&qiQ~X z?*kR{BAG2x{P)(bc=1ksXW+}l*K;FoKVES1v&pR7Gs3%mZ+&RzF?ILvxm>+PT2FSr z+)}%Dhv1e4foxR^w|;+flIfjjY*RI>HeZQKxc;gPh7+zEb2gr-U7*_gWvYW%$Ig&r z+5OxdPjZZBb%ZPW+h1~j`C|8W`?rg@uT_2OTf{oc`@)~JNn1aZ7VVq()hT7kCBJ7s zi{fWE7DqkTii7k!K`P?{sWk-}^sIWf6gBvXVmfr21re~*?zZ3$6!DZuz z4963?I&#DK6lN$jxiKE^n&o}n^7Z!4)aI!ki(fr=z4~wdrdLO9t2f%*i&X!&lyj4z zsg+uBpiH{Vm&oUjtfbGnYi^1Be{UwY)YJ#hwav6E!>$Q)K32Q7Zmvr?b3yv-*LNB2 z+7LhnG}>}ft+M&aTWu4saot)|>}#4Hx+LiAts}QBjJ__pldC3M zr%0BE z#&PpZ_}!cGKkoi?{gm-wxz5GkOw0YZ%q@3*xIjtCd%R`r znq$q+mG$HE1JP-pJp|&oN)sD$;+E|<*_D3a-z7DbIj?j?#JKK6hQ8uv&AUH)%4?@a z>+IvvvVNITs^{-WFWRZ{Lh9_%FLPIVIL`hmzO7JY3Gcd+x};o3&J`8wmhZUWU%Gf_ zP2s_N0e%u&j*1&O1X~TI`@O`IBzkLiowy{FzoMx9TdVg*^9>?N z%getvO_dVUJ9=|&TTjS;qsfIXMd$XPVYSm;r?tIT*)W9J1IX^*mN;=jnsb9US!&*o2k9w#-^x9!fE$jWo~4tw27 z_gG#q?Ye+o;H99|r;9VQWMXyuua#!`as6~ze)X>5iCun%^M0#e_hs4_wlL6J{+)-0 z({nk|y^M~0%FD7+rcCkPqq;xzvi1BP`&7}rM+;QeG4Lu&?6`lX$@xuPiu+x&zLF|M zxBt?-Nne`WUVix*b>MHW+qGSeJ@b5dWSLTH!5mpOgYvUz#P72V>n;-(q5~W zc`sd@r7k{g542T38-D5dn*NOY^Ukc8XSb(c+j!0LUCXDwj_^Eo?%$SelX~VBRz-Lo zyKS+l>Vm@SNx~g(XU}?hb|&wYH=8v&z8lXtGH>;PNxCPk*EMMzvdEmW)2S^-cKYsC zgX*-P^?$=ej8f8r)*Fkj>wn<>boY}zT&%ak7eDw_(AR1#{&N3mhON&JiAtZX)eg9( z)>-%@Ixs4vc%>~%M2qmOx3AA6OuYYXL-^MAtomcT0-IPv_L|u5%IkgRzE9Nj&IkG5 za!f%+5ADB2UJ(gfwoUgy;KZ7=h}-k|oW0f=+$m)YI;8qO*Rbu+B&Ccke_4Kbt^3+zSI`GeP z>*m~alSidm=7I6D#AN1!3$=^WJ|Ej`qAH3#B{0HvdOP})A^U>x2O{>3V$9-y~k{>L8m**cUfA?-&Rc%T6 z=TpUd+%I|5CDqdX!loEN155+Q1|4Xg@F=g{Nzn~Ah`gU%4u60=L zoOSOySEoze+7I(=wBL&ba2y{+qUx#Ech~H9 zh;6Q%nqPC;JNa?6nUwJTPnTZw?A!I~r{&`%o2usRF_FrxdK|s2@Jea>rwNl!^G}gH zH?{P~!R#aJV%p`u{CnDU{dUZ&&5HAOKKkmWY$X3sfG8iLI7)+gjc7PGFDg-dR0+Eg2g8w!b*mJB9UUS+`ZO ztwLZm`$hfou!Y<7BXqWl-`=-kQK#n z@k44DYsoUERh+iqc}z-JS}YY zSz~3-`lXp!GNG{oeDRt`R8myhZYmy$6I<-N)45~snmLzV2_)q!?%H{VPw`yLg_fN= zx@u$QtU2CipFXF^+GMWon(fVJX0E==XOu28Nox9%pP#-Lotvfm*x0i-vf*f)_@BLd z=fCP;YB?Zz{I8t(to`1z_WUxrmEI-v^s&|a=$&O#tj|n(7O(0X{P;!9-W|XE^Zu+* zvahtf_2#pdsh<1nT7GZt(-)uh9PW(#I_1-)@AooeSMFZL-}_N`cZKiL;Qo>psk_I& z9Nlo^_`jrxoYEz3v%D^>*y+*#;(F-^Wz)E0%R@c}oZ9Izv14({d#Oo{&ffX&(i!F7 z967YdB(Tp}>e-t^FFrWCP2cnJ6svT&xZ7{D&D>!ZG=A{evTRMMpY6%{*6`PDn~d4J z4WwF9GM6YXxpC;Y*sqrrU8eVcy_{zBW7*?e6RlMJzcMVh5LS)ZXx`G4W4lJlVD7{EqdLKHlCHVigx_ zI_b^dMC)$vEw_IkZkx8?=awxyE<8W>*J_cx#6M0_OEs3g>mTtkhel4&_))hb*YNYP#llx~&c9s4Rj{haNJr4dA(;2Svg^@M zp^oJj0>1ivT6ON_8kqwep2~A?H2mP6^+EsN?9)J7i&~tuP?6+9fZ-v+1+{y^~ zyLw|*nPstUtnHg(_qft?ho4KD7FdR7KXzy0Jbib%c;%I7JrwV5F+o@;jR zy0~J(iCD4H6AD)ru09{~Y`UAc`kK|}PMS?Ww9Yx-V{vUz8uLZGV0-Kk>(<6&%iyR%_4lGl*^$SY#`{-SH8V&^Nnc zX_Y>?9FzFm2`as-*mnHlyLtP}UyeFOP2avnOH(6vZLAI4dHH}vYul|96HieSt1^M` zrCq*1RwOkPF5PeSXF>2J33qc(U2WO&=-pXj^TYT1{8likdR(;Mr+W59)=UnmxwpFh z&3%4+oi^K|UA0;7c75;)`+ob8-v^n~8{R9FdrdMia^A*fIkhSO`W~)I3+oQE=`kk` z|J%Z#zW&a`%Qt27mVMEzdQ)Cf<9R*)-o@MLKh*CBN*jILx7aM@QLVdd#=*~hw-)~S zE^C_1ecL+gu;RV)D}6@$->;B$th*m*ylunY#e8ZHD&60{`1!lfYDL3yS<^((56&kG zOBB0y>-{j8`yoH}L$2P}zD11ZEc@!*n`e2iiVjV)O>%Z z;#ZmXZw6MZNQ{X;CCT1zcHQBgOZTUYQq#V5+@ayTt9sqGn!BhUeIYNtj_t9>+)~~z zk2CM(p40FVxXXN2QE$b{z1DYrwtE-mJU-#FqT+7;e%Ukq=+jZgohOW00J^eCjB3m3L{7l;+wRmT~ z=603y?|e!OrLO-ekZ3b2pLC*~duDWzgzTIBbuXM>-c#>dvqmXnpVyCb?<~$)+{yd1 zdn?`Xxv6sh?Ojt=|9oeayngOh15f4or6%pq z&Y4Kt%OAB&pK{qQxWYAUO0o3w!`elhQg5@~ao^w7s~o#2W0mHyi$6aG{M@yB&7#OK zMW2ZM8%?LDvaGN$*xkGMk9O|C=(&p(UY8$=o~taIS?YHu!{yD#w+Axv`B!IMKAjim zeC?`NnSJ?-#M7tq-tkZRnEu@5_hrZFEirB%*GZI5*mq>+slpxVKTh`hZCS6Kcwfx^ zMT5b|ZPLA$HXb|l)++mbXUXvg^Ig|x=`DTV=e;&|I^SWt|C`nsUWt6+*HPb~AFg#Y25 zyuUv7)AHpJWwYk6`ufhlP;@`@lz_?GUm=c@|Ic1>a{lYAmlEn6v*heG`>T3B`!}Us zh*9pD`^;-n+qKYR8@~zp&#GAJzsJe?nO2ipi~q@&Ggg~;2HWYjyg6y$7FF$ecCvrZ z+sReE^J(!seDPT2lcN@$eQv#m716Vp z&W09!t?)~%ag#gy*z)MVPUR}=SwVlVOqr~HXv%v3{Y}+IFTT|tS**xkthFV2uiE>B zC0>^_SMGj%;iSakUF`-J7d!F?eAO#jjVP{bQ%e^JqXO|M zg-^6|UwS;cDi9s`(pX^kf7fOHoV9+Fju-h}S9kf|y8gzOLecjJuDb+@y>E;OS@&h( zjz234^j6hx*V@aj__KWVq^g#b29UUcN0kwBDjGHT7f1CFeg2{Y&p> zZqf7gnkwP087KboKPR$x@tO`dA)kY> zqCGm6Jy&kAi~L@_Q^$GozN#;LirJT1P| z`NX@;n0yk7-%{eKEZ@vv zySO{Apg=lzy1T|O`7W6yJ`@WQ`lQc=GuXDqFp zpjTnx^)YOoUWLUZ^I)zI9C6Pd{yaVXc;=CJvWG+-KGZmvs($dW;=7GoN`?2ZK9W1# zZgBXpVOx2CFrUBm^GWRn7auBK-MaI!$cM{!WPO)hJN|AykI_0i)7Id&BJY=PB~5qd z`!|`Y9~bOrlm2CYwl#nD4DYqF%M0ebpHs$`5UX7E>Es8YuM_qb&tSe|ba(HGC5P&Q zR@}6fwk%u3dGx8IT=UD_mB)WRGn`;DN!~(odHr#l^E2h<&#NkX>TG5Fa`$t~DW4V| zUw(#v_gUk;cGDMJlw+Qi9rG`aXHs$I;plM1)*TnEU!7&mcUZr+vRJgLeqG3-@Bc)C zW(8;8ynWdzs+fE6&OZ}sZ&q8MU-iW=C91mrJF=F1cNni% zmQN{fy4M@_=X1=4_sTo-(-dpJzxwxAX={7Q&8mH+P4YXKpICglyFz@e_OI`I+dSs0 z8TM_Nqa67_b4U6bw>5ir=jZ#TG(JBaTkYGs&-b})uajfb^#!Kk_Dz=+E>wQ&bJJCs z_j198<10&!%=%H{UCwDM9e6=Q@X}_-q&4p@J@kELyW_U2d(M=g{WhH2gdeXK``avd zVQqiZ0`KrLrngTO&em30?ke4V^RvNT8U5Z%?*v~xp8h1&ZI@^QqmRD(_B_?ya$a_o zd6U@Iv6s&{%k<(J-=WV1OE2tMnV{NYcy8yCKp8<^o}*9IC(W?7H#4*6yjAO=XWu!c z`$B)=lw-dAg>$C4Y}YG4vUA2Xm+g6Xp3Q%8Ew^jyT|egPbzynS8!P=@=6jUx|FieS z5*`2JPTS)e&N?Q_{j_*s>JdLJPx{mI4*OUyjjb<#1z+jEZgldn-~7IS$BxSW2@8Uw z^eQe+e)*TL#H;#Py3NdUw;f&`5dCC0{mw^+`K?MBS7K&9)B5_vy>Bvm?3c-I_I_XZ zKEL%|CT<(@Xl8!A(cTvaBvkg*p1Tzt&vzunYF(pqM$Kq~~u?OV^(C+>?pY7x|uV*bxY{N?N-JySlv{v{Ll z30O&8cH6GcS0l0W>4QrXYH!}WxAiHHjFSe(lpF4*7Y`fdMpoYr-fgGZltmmfqX>7hYfb*SROB;Co2l@&JC3K7I}85UVVe|e6Qk*2g{#2 zZ+r8F{f56tjfA-Szqgxby*ylED0J{%`47t%d)Ug~%Kn*is_oB#i#va~I4-W0m_Ez< z=CvK)k~jREFNFrlwP>|mDC>9$V(@7vYllWSZnOJ zbba^~lS_Zp4zJl{ec;|@uibVxWE0kNKi~}NJzp!nT;N4#%pY;ac@r{U-PZR$YZ5N> zJYe12kb7F0{~~u;&XQ99^IblM?dxs!Tieb5#RM4r-MJv8YObSDZ9v1fRPUpFBGdf% zD{YPiUSpR1tChKAH{${>!|UMzth>!7z0|0Ql|K2!&$jV&id8eyT|Oo4cb5e3%yFCe z@<7t{wgo@M?##*Kp8fQC?#5o_tGABlKKwBGR&I=?B^#}0mcc==!3 z-leU9o2A^CyDzIRyR`Q*^Vi_KliANlRbrLR@>p( z=VGQ_yBGbTuZF3?r0-2%b3qa}yS*06EkV8Fezq-dP6amh%C6iR>)<^9v4)NIBFD>f z6Wo`-$mr2~HYuuX$Hkk{y_Ys+uDfw}^<-g#gV%Sy)#CDaJ;{X2Jmy8A-*TR%Zu}Q^ zO1Nh(?r{5WBEHBgNpJ1i%hR4c%ByxW`PII5LB6a0(hBnrycT)Eni++=rY2hp96NaA z*}0*vi!G9{3g&g&{6w|Uu z&Yw$ijjHM1DWAUv9dqBQc>T7;r9UgopWL#mTY7gww)-KDtYxqEHOXAbQLZ@8$8H+k z7r$=m%4<_UzP<5(fyupppYDf9$$#Hf)HuCl_kpttJ_^;DI}g0Q5hS~v`wq{c0;^jk z`L{nBJQ0r75j1{b7<~3e!%fC%+)+|ZA>Sh4-tV>kU7SJISF6+& zP2LO_m%UjlJMX$$=8`VK30=qUOBvso%20Vuv#NSlLuJ~J*JgcB*d1LR7YnBS4GX+q zxa8F2b}9G8`yRQxu{y0aE0uN5;_pRAuEk7yy_k3ZV-I${<@}$*Pv7Hm;J;naUo2IA z>%N=vnX;6Im4!>BoZ6xv`0)OgIJ9z>ciH;tJ`-`~GdVvGth8#(`gT?E=7pHW74AG) zx6bgE9Md#^aCYCjo%b5dfB$^=WnGC~yVqUiw>BJ7Ynvx-QmVLi?!u~PL9d?gxN_@i z3=hYj#MtB&doN@rEvbupZmhpq!AoJ@=bvY^W0`;3ol_5f9bppgbM~?1y9*{~`DUlD z_-QKRU-6GoF#gn6*E##-CcgLh_5P(G|IWqkdrY>;ZM-<6%4M44KmDpB-pQ{8Z^tZ? z+t{1@O7NS-dazu}-f!XkH+~p?<%qUhFZb}noR5FkAHOwq-NzS#(!CXPBv0vOOyT+H zm{@+^O}SL8Cg8^1%@;i+zy035_oH&Gck{QV{r~&se9UMGw)qt8mCvTDTN&NO@Q$Vb zcCeyzT2=VAR83Q+xLK3kRtqw;9em}rGY z=dtO>-*mJ4?ThmNcw+5~@9CM+?nh%YBa38y&UD%K!Sv2#u6+N5^%M7s&hb2Ve!Fq_ zp84;4(|>Z-*f^bw`FQU7_NRyAl+Rt?9)EAn`s_`-{5pRTlUu4RF*|;6 zY1R7&6l;F>*b{bb*WR))Q;F}1@5Of)TO{r3T^!Pz?dp>&ISq8`%T~RF3&9e4=f9Tq z-#Gtj_M~@fxYHm1S&)%`)^%Fn|AjS)uk?G*9M-asy82&3`NJ)*(y1+6;SVg-mx zo?C2P-wfFWJ@==*VAJ?KpDX>D{=+k`C)eNozp&xbxBcaksw;}BbgV9UpD}vDaeLqA z*MBZ(oKU*?&GvKIUK67TiHSez`xZ0J`SD;|)d%H#)`V%wpWHtzohUiAL_13N>e7Yc zllzW4E-t;dr^hcnmUpYn1B23QS0`p^SJ=(%&k8@Jxhl7N=A-AYBKEBPa{Imf3Zc9# z>HgoBR;(}zvUKBQ@9pusEPOlJ`gq`syWP*6dfqPWERv~8;% z3QPZTRQmd2!tFJBA11xHVLM@ez=~~&OIThu@-lZ_IM%zz?aP64XQJJ$mhAc3x8Rdm zpH-g2zj-EBhBKd?+PnMGLh)&NIUg$e7-BcQ&R;x{u{7vj+Oaif`J28Qf7;Wle0QEh z*7cH8k+b}llssKslJ6}zLw=ImETttSg;8q{7oF5zdo6Lf+b;&w@G~13-Kys%{R=v^ zfJ-V?wCCBXM@1_S*gUh+pMJ=U=giZ~3b7Hri&r&Gthv#(eY?oLJ2Q=UU(^tOeeCNk zcb=UGWKL|k&wS+0Y#F!6#KePzZ{Ir2df)Qc>e0$AI*yA^S6+;<+;O{gN6ovJ0-Hbm zD09foF1gP7L;L#YP20KW-8%GjPmbZ@dEu#_55$Q(ryPx|X+6f^&O08Mnq)&WKQb{_s=GZ7>6j@jxqY@6W?@c)o;_(AKB;L zZEh5~H^c4Ilb%KKOWPma4Ho&kt8R&8mdxB|w>>m{?@lgWXs&3Kb@AcMDJN$}z53vM zzDB_0R^6hVQHAph*w%4Lv0Ch9P_uo%dt2{`T@M}=f2(!Bx?XDbYQ_V{vKG&ItWlud zuxM@N^9dp?=PR{Cj^(_XFPBsp?wS>^@u+O&q_~HZA60rER?U2+-uleDy~KIf%6D&9 zEs_3KaWBO*+}lO?Zhz*{mn(1n<+@f_dn&Y6^3uzN{K4N+ZWL5i{rL9K?CpfItNQIn z%%dl*47jYG8)kevBV}jY43(y5t35A$ww%62{bjMJgu(AiFG3DG{`qiu{u*Ac6ME@K z-W==i>A&ziOYNi8p9>G?Ol>!l5|!4{ZrgNL_3R3#wk8R8=E&`3oo45Z^5fMIsM3tOD0Pu zl}1`zFOUi{^IgU5oH#%9L4ZVmLbv6m+h*K9bAne21VKP>x(6LI)>O^a$0q6?nscx?|6XW&qOeM@QBKCX%#>R;k1Zu$-YFA}S9+{2`u)V=4Kocj z(%04LKG~!!FJbnxC_e2#XS&asd*xao4-9+-RTKGxhcxIGP)k8d{*?_;Wa-Nn(zA>ba&^AUU^rU zkf~YiTlPJX+c-;J`*cWW;$|1e#m81|S$8EyR3x;$BmaBOf>#$Zp3aWSaKHA;{Oq~y zf?X@~_nRH^Ui>I}V#%kPp2c6DG#Dx6ED(KNpu{$9N74&7<%|1Q+T9WC zZO+oc+qYLPTi-m(`%}+@V^?m!-O2T4V$|Kzi4&#Q*Yqqd%;504;%s-HS>eytO+OoV zooKC-kyK(($_$B|{OhsBfrTpzLl{~_^7Q<()^n$Zr(b-WD4UT2{27?9Y9&hwuIU;o0MK=SLuKML>Xg^`pwm(c2TBH1;N6 z<-2ub;XWYu=QRhfl*4#ZhUtuP3< zcrVCrPQr!#o7XKbJm)j$fvm!s)x*a>uiSB)DJp1*LV*L=yS=mpQR++^1e^xC_ znzrHYwWVuZOkQT*yCj&&-6OYU`n*p}idNeC=I`!GF1uq<^E+rs)pJ%0ZTE{;NAdf7-Lt~> zr8^JREt^oB!@cyyhUTubF`dWml?0s;_qr&_o^UgB;>|~luhw>~UU#VUs{NT2kRCD20wgbz;JQj(K-LMGD;4{rp%FB?6JCc_WO>&kTrRCHaZtS zopakNx&2vMVV<^ld*L&$^2+G+btTicmp)?(DLcmJt9hQy_O9)U4K?fI{3Tbh*}j~d zZ2x>xZLauGxB6P%*1$5!vkeDQO&>S# zY)#>>H%*?Ne|SNzp5!X$slwM9%%wvAFVp|B;IQ$m>X?r*?q=#v#r$skc_x=)RD~y9 zY`)6#cHh+6)sjEzw0_^x?!UgZsp)FZt$kB#&81SE?!5KLDZ5afVZ~=w7s7vjm(c1t zg)i>r=X}V2yyDf&4X&%I5ACoNobA?Eb7780pK|7lcjw}irF%DpA2a$PD|u~q(ne#y z-O^e&w=0!>k&b`R0$W2sDuVdH8#R|15@}7%@wzSRio_>B$Hpk6ucjF6} zsyfWvA<=oENuD-*wWlHiK@3gSxyWd_ioBl*{>8u^OorjCv-p&`YkyKN8 zX2^1RBL9?guWJ^X)D`GdEOl-+(Q{nvopY!x=ij_Vz8!PCmsTCzm-Ei@Nyz$|eV>B* zJs;=YOgb$oaX9kwiMrQDMt^oJyXdyq=-A0cs}D|94BFDR;9?su&%I*3xyM|KO+S9C z%jS@(nKsd2-s<058yuC|19=_ZscG?v7BVRnqRPH{TigeCvGvMB8u0 z=J-&(ZOT6`GHyw=uns=XqDc#qKf@XPE(yFjjE$wYt>t!6by}lwaf}#u z1=HhBdWoBy`k`36J+-iKNyV?bUtG^VjTXwfC9lJ2yy5RQxw{!65^p!EoY8o3`(D1Z z`y~Yzh2w#Z%QiAqos0P(`roBlEw9Ny`lcSM-Ttmv<;RwHoBb8LZi#y?HpvUQ?en}(%jVXF;**5|$?koN z7jZbBSN;2B{juD6JSOk8gNw^EKIJmb_#wP*l4gd;H{NOUXLuhM49NWQB=UL&ll{jh zkykUC%Hr?LK6BM<{j}J+3({?c_0sl6hFzw4`E0d&MUta2X{^V6}^Cr&ClP}l%k4hLxo=xZ#C@10xndo|DZmkZ5PMRm89b9Xv!&RA)DZuK2| zlU2?MdskOmTyhi)U3Vuj{y~8FRPQ-k&!^o~J>a zQT6U`$#vEDUM9`GzkOEZ){9mCrkk^rR$3^pzQwTVTkBN)+Km=2@wqm)8)hV4siw`lH?l|7R=m5PEPCsH#*$-lzqPi!?_riLa8vfl zP~7wX?;Va<-nZ*Yc3sq92#i?DBf}|XA|?KK#kI(n&v#~p7Oz~oI7n`K{D$(mjUJby zHXgnFzU}EZ#S{ghbY9Es`c-?ZZ0tkha_e@uTBOV0EH;ejY>fIA_*CxU9N`rCt1tiT z=li)Ls9=SdacEiaIZfsjN40j_iZ|rM1f7c%pYwl?Map|tpN`0RS0k%?SKF*M7XQ+? zddKOe_UWh9EN-{Hn-udtVR2w+-1<=A#KnOf%Qj{y-B5a>wSHrzPVvO+Rd!NRmqNmW zE|^{2uCzXP$DIR050@xZD;s?|vU1by;HU1c+op%+iQY_GerM~WIa9Ci4x4khXm|Ee z%VNf^l1+=VM83Fh-Wtv-a=Rq=c5v@*(Vc(S91zfSv|;B;FDd8Wu-}4J_x{dv&ALw8 zTvpZpebK#T7yFGY56{)r4aTAScC+7z*IReynCqvxOT{lmFW;A$bm6|!FFyOPA9HV? zew6h4;_jC*GkZVRzH!gHc1wJHqmja;Q{OlG^E*#m_i&XPkLx3|xNnyd9ej>l_g2_`mP-+R8&AfwRAUTJZaAP~h3M z6~&w~KNobZTN0T0x;!z(-L6BVU|*y1{EsVo-&?GD^>MM;tm89h8<^y)i9DHA+Oa(_ zx$dTyNx*)S$r^3V~n4>Wa24*RbyYT^4(Jwl0|w;=bCRVdm6@P zzUGJs|0d2h88=o&If>)mi<=nCdk#-AQs?rKX#Xk7@NAif)XjxWTR*?^nt58uKYzpC zFTW~w=4v`~KE8I~+^!unuRXnYX?yUrzjlgS_kxzL)UDl>zKut6rgBC}|LPr4ziusO zZ)CEZt~Ym1c&};o+s&u1{@giLaR1Gxt9DkOS~_|8r`*qH_HH;=SR(Qu@X_0y^ZFwD z=DcPO*!cZ*=%d`9Hg`A3%vvrzZL|1=3ybUTavS;n`YEOL_V>#5IrqDLyUpkGs)(Ga zI{$5Ex^CPh4R*Uo=H;svo^I~-&$$~K`QY*Tz?xg3AN7MSWPdr@ylDNM>(|7LfB#tS zeE8YSob9vLKRDOA=f&!s=d+eB7X0*MS)Ya7zkKtz?Aiy?_068!;!l#*&gLk{ z8~-Mi-`_x@TX^yktN{U3iU3LVN; z75HSyEcM^qUMKS?Fk<1C9h=@>dHQF^rnjXBKg|BM_iVA_xx(IeF?VFv&Oh${>Q07} z@y2&$aYgkP48FxYcI>^e^U7n#!#6lLUw^!8n{n*JsOP`JzSTKa%g5Y5W!J`eV_m1t z#^+3WR_^nke@VD<*2PeT>3mm^bnlgEg=TKwo;y9C+y`<*dyhW?c|#nrCwx9#`Md0VTW$hhWh&BNvGYxfFgR2$Vk z^3y5^JX4c<=vqzPe7o)ELROz&`&XCoP`cxdvv*Uz@NF;J_}=o>FEP)>R(BUkzgzL{ zkamOW@>vV3K09p}H_6g_@y%5HJ8GKv2GCWc@@lB!DrxTxlKFfM< zy@*S>>UU`JPGx>Gdymg^HOfA_g*@r-pMCV4&$+MLBHxrM_SaPXI{S#rfMH$XE4N)t z*R~d&efyqQ>-V{s;%MKsv7Z-Lyo>)4`18<~c-K8=j2P# zIktbjO+&9N>srpuJErQU>U}({xk|dX==5Uo#Fh8@j5f77K0fR7$T8|I%L?mD2iFx# zDpc#O_}24mgZbaJACmK5L_a)!uVee6{@U$3YWTCiN%u1C-V(}w?Vj$jE8mOtT7w$X zzfS2Z+qpbd;fD3Wyq_=R9KW41-f`rf&V$6X&hT{g2UG3Ox8^Mk3`@P%n=JX~!4eOf z&EG#5uQGGD5s?%WN~ zyMNdwK# z?dJz{&tLx+^6>Px1B`Fx#hrC=ZE5(W)S+c2EGqaSRqMFIs#mG8qTgn7mUFyZcj%?h zl=oV5qX>W5y1y=VsH3oZKWN*-uWlIlNl6UTaJJ zsq2PQwR$?czQs&GG?UxaYo}go?Y)J~9(7Yb1su=%{q#($8{3_4*^2|W+P=#A{d7%j z%<6e9wIRy6)uN9xUgZ7Y`S$Z>@m}4%Z1uXWiNM%uZ53>ydm66Aw*B729CzWVh0z@U`DjO^@Z<7Trl-SYxz(&bIF& zM>(S3UbRk~YUdevsph2dnU%YAE{A7_)ohun?jYdy%~WdL<@>u=I~L?`47&5SiG5Sy zpA+}=uDs;GdEMwivwj~jEs>U4Tr>0Zx1hHC{GNQf znHt-+=O6H1{OY*<-lMZRZ$H?_*kh2VmY4jE_ksBVHk;$elQw29$TLo4B=mJyWL>BdTE(#{XD_gvDg&QAq zq<;yOJo?Ns?#1jF+i4FE_r^FKNt3j@6DfB@Op#|r-MYrBB#@Z zj0xY8ugq9r^(5(^g}BwJzb4l&o8P>r7g9X)L6&>lTD^j=ZpuGjW{TF7A5+ZD-1evF zT3=t$@q+;iUR;ejBbO|dxUr)8TcQi|EHB4d&Hr9x0r&FgPS} z?Clzx#q84;1g<^mk-Tfw@1}{p`?czG9b2`)eL+?tG`b)itK&?u;xm*DjE)^N>`s zw%c*Zk^7#&W@|o8=e@nf+g0u~#`SXQZz!LVyJpht?Pc$h)a+iLI?)taJn2NzQ_WL> zCQXZVWey+fIrH<(EXmNAr+Lh|`&X=37h#cjG4jggBWGs5&P;lyv-9jFqwEPQKZ6nq2Cr`?uU1McKCbL zD(;!hS$T7jo7$e<-lshSn(>Paup0ao89j%$x{<)7={4&lhowsQ_Q(NHk z5P|ko3Ag74Qn#I~uDi^+5qvS)xO??}lTF(tZ`;h7kri)$W!g@bv*rcWcIJPjx7AKF zE-Yhu=(4UnFFt67&E|i*Y7Yb$+ZFe(p6+J#I?#Au&ovuCH5oa*%bzZts3=|3o;oez z=8uUpzU=TZn4BEaJU>)eHqXO4^Um&@`^*mP_gS*H_Ug@uhxg8Yt4vQ`+ViN7Ir%Bq zySWUzQg4NSyOemiXw{0nFEzAw=BP{W>zTK6j#_p%yQ?az^(u$tKZp9Z@IH#Tx%Qjx zxtDpT-)!))-sNCz<~7HTNjNtzKG0<9D*KBIes5h8cw*|tfY;v^b4Q$VS!hj%Iqon_&TCbeFs?D*}b>q zpK|T$tZKeM!%eLh_eB3P3EAcvaiXX9;{Fq}b(5U0XaAd`^@Ta}@|GP@S(Zs{va`== zYwTs+;QGX3msCn*Ym|(kvfp8Yf@+zN<~h&0dOy@1HhI?FpmRYn<4Q>WjAL%z_K8xn z=lWc;Oz*OKlW?82mu>Rp5aTJI`;;9&ZCzD(Wm8mKui^-@AU7=jD5P zqHsyggS`*TE_&JV1^k(P?b$8uyT|hE6=&ovvb%rntaNY4+sC|8uhr~6@gCe`^2uuN z!pV6_V)v(=mOd+9yrkH)B3!|hX60! z=egHivUd-AZC?Co-f`Jun>8tyh-oEBO`iw>?g3 zra{M*>nBdm-d1Aq=-|tZ-_|LAEnPP0*6z3ll_yO4-#(vfV|(6l^R-jr$rElZwO6^n zoSYCV$@I3g@N(?$dmBVQ3iB#SHMyN+&udk@!n0G{LFm~7yH*d2)U)>z(-jOR*>Yaf zUTb9d&F?_ljH7aItpk~p)DPa(U9#ld6z|`{4RwLO9FbEeIlgd-DV>_jmHOMtJhm<2 z#j5cC!9m_pwFTP~L$p_H`1Rn{E1p$nJ64ultehcm^lDOIj#`&{_~L_~xVkMay{v6@ zy>hdV>rzedm0ICLIc-~J3Z#`co^3IR{!nlwc@}$`@e+a4pI+sgoN8&%JTyOkw!Dk& zjxAN@)&d1rn)m*?9iwIBWOKc$XR+3e%`**S$`Vow>_ggR9ZD@uyx-7Vcl>hGhNI0& z?tkZar+=9i+Mg!vKJ|6wma5H)OhKu7j*BP8^&iXIcFN)EU&V|qtb)^C*Gs!c2^g;4 zS>ijtBAWf#tz^E_-}B#xpHXq{Q?z)kRX*q7ME;tEJn2P08aWRZsI|{8tA6`-^_es` z!RueVedlwg7d>;@boh$*-E*eIu*dDyK5@*0_n<7B6xZT*nX-nXCy&-175X!0PW8haRs-=Sw_MhMs5$dD zUOdrpv~>3YbE(v20ujewl8(%Q%$VFe>Sb-I9EE&?*0Mjb_HiKk0nuB;&uQ`0`6gC#zxbWK zlkw!%=ee^o=ZA)DnZ0smU(Ktzvoi0emUNw0p8GsGNOd-+tb-Tg*4%=g%H1lfT9D4BlR;zGH31VQ&9UR@#E6J+_o} z#>$(d6nE5_k{*cYP8vE!L4R>2_ zzS~MJmak^M-+p%K0><)v^Nqt7=*m8Nth%@Gi^7?rL)+#s2B-1N{A;Fs*7(0e&>|Vv z>vj7MpV&5G;sMLU`}q!Y{J-{Dd*uzsjPMH@jFZ{|B&BwrU23)CLX7UN?fhHk2cLN7 z>&`xXrfEj~MD4vFiez|$)aIRDxh6A(;iFAWD(}~aLdkD(elD1Q@P)H zbHqmrv=_K8&;5UWWyzmuUjKxjY@eOb*brmZ_A1Xr>+HlYzp5r_$V;vKD$ZrM$K?L& zBXj;}sb8FY@W&;Vm+948>!rW`N--!CzW*V`z)Htnvge$^`fGWMzMHMR%lO$YYB%qi zIwOhQoNp_PCAJreJSa~!xOKY9SmJGSW6)J$$?VO0SZ~Bn3l;dsxnXz1{nVa63_so; zxA0;-*Ke3~IPP?E&y7D02Os|`)bD-6c)H`hyvvV}*&F>IM&}-rTz+>#?Z;y|3#Cs~Q);X-Ll`&;HZ(QKI(Crsj#F->sTXoxG z$tvE6qddK{&p%xwB)s=<^Q3aqPIKp_a&1@D@75UyslPR!li%hfuJF|$rC-+VbCIl7 zbi)e$x$gS+?j{|{FFo(dSH5YYSco7BKft@6nmsPpxa-DbzjYnt%H+ z!w<>z&8d>>&o#uAe4qIEiOaDER(G}>f3(qjk-b=dGiRMlJD=K*W7qRcxZYQ*hupHc zzceGf$T9O0lF!^;6>jeYZ;x=KJ1#wjh4prZ*ll{f-H6O?&rCC@eMJ-R|$P>sGH{^+%?BHqP68qGng1jTN`h z$D->3Pc^)3`}!kO>?GEFJN~(P?V0u(U6a*+i;Qng58U|e;0o50ow*h5Gr8Aod3`6K z^_%Kt1()#TXE_HZ-F+_mbdL7-;=*GG(|F^1a}6#B{rk4h`e?|$yZqOAB-JK8;yt|I zMC(Z1!4`&Dc{2<+`K}fp{+f6wZbFTP^g8Ru3L*S8Cys4OQ9pX~K z;WI3dx^H%Fy1-$d$I@!Inw)tgrygtX+_pgdi7@jWlfDwEKmWc?SkQOqyXeRJ86lTt zf^u1Vx4Gy&S}vbGeMVN_q3%KxIf-e3tLDwQC|dlC$LjHelns>$r_~SjpLuq0LdVp@ zXC#{3%Fiy`n;@JtxnFBp(~ZvR2bMespB*%tdP$b7U>WUF3hek39OpoqW<{^K!To^70geY4j*ew`~iaZA4VtI)n;3E!ab z1&87)oBp13?C5>weR@t^ud|i?(b~%=Z{_d39yVo0T>FBR(L28?JWYAEdDCz2MTho% zjOt95?yZ@1sn}Lt>MPH)-L)Og!52QSmea`&2#$>Z*z0C@S6AoA2k*u6EG$xvTlxj9 z6t125>~_p2OV`OauKCZ~Zz2)>{a}ps0gufW=jo~@|hlaps? z{5910B>V99#`2~wp$B%KmknhMGSRbg3tZPLt+}!HlkP*yxt}hZ&dZbeV%AqRJ>lGz z$;ngC+zHd_u@#?HS-5^xuEfth3vti+!?-czy5-bAsp)^WH{JVuf0>`b zforK7R!zz`ky3WD;(FQ6-uuaOy^)A?@2N;9zv&Au&PoU`e);IB#zXZVftB&xUYjnj zc=S|bec`1SkDh9TPF%gaY?JlVm|Yc>vo}ng>n5tdUU=K1lgr~b*Bm+5|3tFR^-ArB zC&mqLmb9OY{Jog_-Qya^luddsPCUQJS}wIz->LrX)!XG|vD{K$x$bS%+EaRWtJWOj ze!W+MqUU5K&pC&x2NGY!Pj8GcRpiY*SMHmf`QVH*xxH+Zg&D zpM1Wq;9ah`*2dOZwsjMF#9qWzTxVJ$R+9j(OP2Oqb-whQ`^CPHf9|;=_p0dk zV=uRSlewcEQCJscckSQgZZ_M7EhlQ7wQSklw60uFo?>His{OB2&`GN~F{|^Xgiqc& z!ajNbWsk*A-hJ3|F~jFy(Z~0HJd`!pSG~0EJrxyw)T&G>REAly=;xD2Nr%L;*rLpa z+wb@BEIhDi@g+ayy`JTTJpOh82~RCf1#fv^Ykgj8>)L-7*DteX$-9d_x@?nOb6)Gk z!#A!zs}~%%>+|yL7G{2OamrM;=Wke>KOH{Vx8?YRwGsN?SGXyd1h^i$*>xjKY;JK) zK+PT{=X0}H7n^p~ik$oyJMYd;k(W_dHDn$hDBXIcdh%=$x!^~&HA_y(PL^^He-v$5 zaxS&rtNVR+_zBsd)B1&;`xm^mY(Hmd^Fsdo(|vQjPkCk=?quir`}pw5zEAg=7coA# zuIH4QTk=Is>v-A9f3DXT-&$!cKBy{i8aV^1zIT~4C3l}vO8|(n%($5&WIpcs)?s@!yZ?raan}9XClXua&&^)?(6-+)NiMkU zorl!UqIJj1;y8~QT~G*(xtM=L)^YLboHbR~&K;bw?3l#78QCsP2P!r9nbxU)y;pls zZG*kr^!a-N^!n0TtfXTvWPGUq-y!b&y>>aX*`LLMp3Mv1&)#1Wq?gVA?l_C7$+{VS zhX1EH7u4;mKYdAKnOd#wtlEt!|F2xQaeQJTQ^sVC-t8A_SfBZ<{X6%yw2}LT7jc{V z)BazoX??cAH*8IDyG>JBVaJg;aVIm&pRb!v%kI7Lz0~t`dw0eh6DwDX6+1&V9gID9 z{ueUH6PMvJeRsF<;m7x?Cdx|c<4_F%kG85Xhd-^_~=#oB54w*zZt z&18CTc1jpmL-8RwnF+Ow4G!J6Y?buQ&nk)Eb#{YczW8-E z5fiH`MJsncxp^Z=_u0({F^_NTD)+W5cI;DrET3<4?Agw>UA5=J3%ZZrsP3}T{d>i% zWaXz&#q*NvJC9|~GE+PMG^%vl_nj$y1v66j?_GI8>bX#OWK`gi2|SLUw<-s5-?g^O zxy3OnpOwY^#6jy%IolPU9>1xVCvLrK_gk6lm8KQ9j~?^&x@w(tM{nEQ@JMkS-*S8K@0JhhHYG`9 zN-vtbM#gOo*V~lSp?AgSUGbllRdYFfj!tL(D(;E@`G0-=&)AfvR0!I-i@J#yblWpA zn|O^)O-;G%>=dB7E9R`NeZ49bw2Akx$X6!Y+uylc9NteZ{#Aac(Z<2U+@wTd?)UZ= zn{AJ6SgN=-^yH$1_eoQh>D=6;6Dj^XzVF|k=H>sd$kqMrumAtJ{(qzX(e{rI?f*TO z|Nr;;|3k<9&oY<&eJby?hV{cg@qd4|*Z(sG-GTe-xc$G)zrU{k-+sX*{?__ur}f_# zF|MkY-0^YgvCcp7CL1KYq~xm$&-=@=&8lD6`0vke`Tsx6zn^{{U;lr){hy!6!dXZmce@>-xXt*E7YxZ4o;6nipH@?2N8qoo=sqtfWU& zqG!3~nw@gVrF_BmrbiR{T#c^n)fS6aykK+vneV=&UrQcL+PX~f;-4u2zEwd#zq~kC z`K-d|tUvG1hL1eUj`)9E8I>)i_3_-Y3H+=VU-&#RZdWX-vWomK-JU6btTHV~qUz@l z;TCSKz)zk#XZN^=+n=2)ZMo6c@z#gO{!Krvjy$ltSY5rs!RP6XHJm+5!mIS}OZHRC0ECtE8HaCw(EWNB5a5mL{f?S)rHQQx_u&ifM zUp$tdv=p2E>`G>x6Ngy*%xib2maNmX*4(t*G}WMHZxP4*cU>#@dg{Gc;_*y#=HmY? zPyAiKtG6$#%zKtNDfn;6rB7;>DpKoib7@V!){ehk>i&$6eW(%S?O0Cb9?7?+xCSE4^@6DF5)`bmu~Rq&Z6M32M@U&+nz4pmX?)o zC1-DE#NBkZsPWC0tSEogXZ&wdKcCQBQXRB$?Np~-%XcP3MYOW4xqdn7;pxz#pDumR z^JUtuDa|wCp5c@%x-9r-1Z%Qo@m=qwU%ej2l$YHS>xl6FW^?)CQ{&G?H{$f?UR!iK z&aZT0pTq6bE7#S2I4y2d|9AJJ#i9SzCTaZ$xDh)$;+D1{U-#zh&#yAqU%0c4xviRG z-kw=6CChT zrih&E#h7UOis#7_UGGWsJlK2awf4&chxhIaW07+-KB-n|e=w@sVbe_SGh$~C9$svn z*dqP#WP!T1-itbSZ5CV9E!F!NUp;X!>3aX{%Yi3l zFGYTyOYC^^YPDt9($wR*^N&_7UVC|uZ*g<{nGg})w<6|4~8 z&1AVxvY~{dain$4nm_Szdkoo>_xOCed~%^}*~;(RG$-7W?q2rPVfGJMspnH$&Sg5X z*C|YXai!^LO~R4JH~P%q8&5uZHKQ!7pIxr5_gSLG@hYC9`EDS*;L@b*Bc=J)Ju_zBo^<;IM|Qu?RQ-=% zM73`u97(z+s%>y=*V?_j{P7=d-@5%_#kG6P$~p$O9vg=xT=2;&EjT&5qRsV1?!FW2 z=WT4_ZSziWwP8Ho;ppF-8u_O5eFyWFv~&LIndt_HxEA`pIki*dP)1AQOWkwF-_~Z_ zoFsSW-sXudoYDzZ$1)7%l$cm|Ms$k``JB&~Jg4L1vo25nwwIE-EX8C6c3Kw)w=i8c z7Ac-~@~SUSTlOi3b$M%FE^ud&Nig~)b!O=cldxA;g{~CV&kx#crupuD_UdQHI^`O> zgB3Zd)fI{YcbVEhk((_0oz>@z5dU(^*l*MK6&5tU-`|qS)c;^E53`VAaHh)%H{WOb zI-dBs@S9(fSbeE#>y5LO$9U^B%+AUl?2b5F*_Klvkr<&AHR0@6FF8g&Hsv0-sP(s+ zwl4e3bw9Oxy@;Tm#nGAf114KfRK9(x{;|Q4FwV14dYAl4=A{I$Tqa~c_3(xpd^?4h z`V+Vp1~Wyzu&fd!y<93vop`ldEkHbkTTcfY4+it zbsKB3bUrLWrEE>^0_GTI(~&ZJ#=QNji0`bDaf=e&B6 zqIFTF-9?$T{=q}JB(>fPwTArf3LVe$$;xp#EH#+Utv@0-91{ zRuxOw*#=0SHJzMt!qv)DYrB%y;+HSaEn_b_nBbDY)Hu6h)#=i}74dERC*}VNIJw&F z=UeAogFj-wzb%xLNc^wNp0%S-TCQ$#cXVKz`;GsM$%*nd28Yf!)r%~2=G~|C$F$A; zN5F}!0IIE7mkFpEUFNF3(SI^m?c{fCrC-(!PES@Vub*-dF=|GrG7?L7awE5&}-f>&Q!^QCY8J(+kyuk?Aup}g;zKa+k%xu_gB zlz3-<%;S;GAJbSqWxeo|j$59wdj@>o7ZYA0c(QQktPG+)IbVH5n&e z`RY<{zFdAD#WHr?8ol_#N0SUL@h;xjb~|T7+#Z?k z&2f6)Z%c45oL{`SScl`!mV{dZl72NF%icWcyRsumtC{I$qQN2OhV^2>MjtC?rSd6X zT9UD*N~)&Mx>lL@eaMG+*1cXe(J{B3JA|x4*lXhoI-7!cLqdD<=la)WT>S0H{RlUYV{!bsDwkm&nIhU<% z@3mj21pc#G9DDLJlc&L7UuL8Kn|2}F6Y}wK7FPC}CIxRCyPiouZaDmLvvFn3EM^wP z+4?n`L6-bcNcr5F-^$juw?uPReh+isTAW6_o2&ToY+_y1?U^=qXBqEkyZ^cT)48*5 z!uxxcN%QYzCfq2lP4B2@mWe+3`LYDV{5XrG`8W7gt0vg*+fz_d2lD9T5F?Nk#xq;e z4lft)`>|Z*%l)636Xv%}t9%%}urawj=;R)wcP(YM_GcOATrkS7v0nK9oPTqT`j#`& z_urcsyy-Z%$MDF0#y-Q!x>@GQUtYE^!0QQl)+INfXAr7zKNS`o{() zS57>=dE#NRs8sfq*Lf8ce0-Mmy@&r?{T5!c*80ce4J*3eot(Q@%=W2##Fq#BLW_2+ zaVxNwF+Q>Rx|5BV*PZ8!mh_$U{ikFk67=tEr(;mfjNB&$~#}pJG>`GK7HHYf5#j< zp8VP6bDY=s{?VExAy4$4bs9(b?(23f@{I}ozALD+^@Qig7~Q(p(Vg?c-b~Fn@+;in zF1L*lV|$9}@vO{wEGJ(qw|jbhv4*qMQn_CNoh9WR|5CZ-zMtN>J>{g)s+{bitFsQe zG(C(mS=l-D+p5#kW+=HXFOry7IVE$2+^o*!6Eun?&WUjz(~&T|v8q%wUZTdd-ZlS< zGN<`r&aqdoS2{1;Oi>ENAw_C(LAYPFLuWZIU>rAC$Y9XG5p^7|^O8B}>*p!b|# zXFnTF^KJiCUsrm}TO?&zCK!;Q7 zH%ZT2X^!)~oUJY;8tn*e&&2o766dP_olvf9rjW0ArTIpK(s8z@ z-?m1~F0pXfXr8@F`R0awtofWaRUEF{CjJ-c3;8e7_k1Z^WB=E%Tc?f-@kwfm*-a~m zJu$0LQMFhnNN3;gV&+e?LN-;je0unmb*J}I#XBmCDownNUnq9(^xf_o-|bc7y6i}W zpGf2xjxlEqJ*RNwXd5&M+C7=7jk?q?}SN|xz+aq=JM$}dp(}^8dGrS^{ zIJwn3Bs$N`jI3TipXtP*LLRq@G8LP|s@zozmc2O=vFX;rwN~{9cgAgvdUg6x?O}nF z_L;3uUQ9XrJtiy9V9}+IyROtEq#QrIMN5`d?r%%a@eTLw6+ipS_470S`uO9(9=ZP8 zTUH!jJj3dD#hl;{Y{%+eB%`;CD%h{CQeRX=C;liY4lXKmFUEI2VY0tLpV>Vt?C&T~T z=5b%o#o#y2>d(y`FCFx*VE=z=%}Uw-za3AvO_`Xq)n?jR=EZH0#+MnNZR`jo9^_yh}HyqEKQC|J# zVbu2H`)rLb%l)1^bLX3lmZtVOlWZ>^&R*|PI4zK2hbLRU9k07#9b?msJd1DB3>pPe z(+m!=eEMiK`A0~;fyp%f{7KtqMu+byJAC-=FYUP1n@#hTe%%hrGsvC#UsC1%cG24r zcUB+wJ)-gdc7ltiK%Qik!Sb1B%+4n~@pYM){85^BM((=Vt4-IK$?I<26q)kti^8&K zzO%~j=6f8Fa#!xzrqnjcF!p%6&|ii_+(#D`f68@U^~-kRho0X&a|^e;%2KXbbNXnp zU}8b*j=9+fRV24ZD4VT0ZGB6x=Ru!ryt3-JR_c2s=D{>qV;P(zqr)@EGTU&|{A z3Yk)u>%SYHuMLcTR?)xi4d+ zd6qB#CA#BIg3IfB`!4M?tD2>oxOo2s=6|{QiC#DOt+@TQh3;MDzFMYiTKITQ^cjuM zvYM@IJf~`w-#wvv=7Yqm{B!3%Nj&@VXT#xw%Yv1{jCamItGb}&6M1RXPgf(ehHX)L zuO@MYCv1x`VQX4x#?0|6V&QtTj@uD;uGPM&+57o=^7Gdw_GSD0K7$zj9m(b=6@%Zd zKKt)Up46KDhGhHYAC^9Rcjn18(JLJmed3eNBodE2X8YbUvG?Kw?kCM@Hzjn}O`l$J zZ}R%rPgP?-E_-k!;?gDc?$lY=wBGw?E!!8TInnj`Y|nFZ_pbC;oagbgNJAn!N5k%P z3C|kqI9cADtq00>W$-inzHK$(qur@Zvr3O8zC0hD@YTW3c1=i7^4Z_DkN=dceWbhh z0dLPS6)W4DORqj~iT}*geSZI|H@t0W%UM6Ka42su%J2RXmO8OVs$NQYldZ$9yaQKj zo2wrjNcd6JTm7IwN@r8+f!fLI*0Ju|ayD%;+xx3)LoV^UZAn;M*>`eE$d=6RcT*}H z{oP;bOM7oAa+cC<7CidDbnmjSwTGAg*uM04Tc)r3^?o(6ONX?$9&TJ~K4bcGon^Uy z54=$P%j;(Hf$Q~^dkHR&b+`My?VY#&jkiP||Bvr)Pg|a~PfxHjd9nQ>|C!ghU(AeO zDT`ry?@|3p&N0&-nu+cAbem8f(RM7tP{%&tetbns{?sylzj9O`W-Z+3NY~?hfLA)ZG)B1;x}NOL*Mwc^qC; z{JdyyjH)`DvaU}=|Jx_s(Pk;CA2u#&^;A!-y8ET7E#k|2>07r-UY?75dU1C8WmVmN zUQhGQ#*f)Pf9mtP$Cz?_`<6w%{5>7kck1q&P5AIqZ-4&b!^x_OvCFSA9}f1~pnaV8 zoY}M+g^9M0z8vV_QAl~$Q!v?)|B|uWjVx*JCEvG%%4eKWEYovV&iX&WBQEs!C2N zEdTZO#S}+=rkjr^F3;7G%U7Cem(8(#@6q{fnc=G@I-W1!EAKclyY;YX&+=yvk4etE z&hxCvUQr>|qqrgYL_)=nM||%kDgrJaPppVw+jy`pFJR`4;;XN74Gu-_e)VD)?nfRaS$WqMo%h-;-2Y}qFPCb% zUby2rvG#{^*By7Qf1xJ#E4%kV#5J?UzwdwGqb^s(O)^KU)o&R&^vwte^J z8S)Yjmy}AdygZ;>#dSd9k-?&^xqEm-Wcl+9Eqiuoe~*+f(RLw;PG>j!KRxw1%U!6V4YO~!g&6PWzKdor- zESdUII_K`R=3P;Pg(OFmv;p$O~3QN@3qs^9wz7YM}vcYo|MSbemYaO z&ghVqg#V8hzg8+V@1A4Q?Nq12RBSUxKY~x0-F5o2pg9k`_xa5YQ{1n2cIhLo?Pb|} zxq?_McqXUqvC;it@w-BA`GvaoCi(H|2kZ@K(6c0K-fWk-tDma{&Yc@5Yn{(Idzxh1 zs~aZs4&2NTGw&?8>n{=M*|O`}Gu=oz)h&-39b#v=^S)i-ap^g`>6Xt2Ih*H|%zx_Z z9qO$A`%FU0;$J)DCL3g|OFSCIJ8O3I#;{uEa}}lfYV+j6cgw~eyU$Vn|IT9 zT8VjeddR0_&&zCHXZ%Cay`}yA;o#KT5~))>{N~9U-aO8=jy=+r8Mw#%V~UAH=gB#8 zGA=xBZ4(5ypIP2f`;hP1uWjA?1pD6F&wd$k_1(wSXHT;oxGYl}&l$qs<1yW3-p3se zO%swL&G$(2__yu}mRNo3l-zq2-}{A!56@gQH_q}*_+Giw2OoWR-t{*qmjtHsvv|`ZdR?qqg+U#dOODrAH@D(K>eN?EILzVz%`c^8f$b;*>1)-dnUbyl2@2 z9=9~HxanGKGh6sM%bTDLRWlL{q?4B3i|6}$@Brp6af4fEL(bPX&3nh}5YOh(C ze@K1V?t>q==M=@3NK{NPJCS(r*OWPze%)H0a>q{MbnC56g})d0KkVR{yIOB`bA|?+ zQeIlZdW+!Ik8Y*fx#S#pa4*&7$BOJwm)d8K!++KJuUymK@wL^#`uMrTiVuq`8r{== zAAHvsyz!#)yZnIExqLRC4ldZDl~BUyd?P*Vz>7n3-1dbT^`A8Pv~`l^8ID&+KAzZM z*S>hs?1Lvx&D$1t@sjAF4Tmk{@-`e0kY4Zma!>yS|L+GM$@>3QVN-tB=;m8)eBJi> z>PczR-XU*~lsYGV2wi`zj_vH%RZ9#LE|~Tn?5K3}N!ns}<-0;&)3eh)zvOiIOtu&M z#;>Z6DqO~-FE{nh)e!s5zn0JXm3~Jf^ttZ7f@qN%i`04F-H+r3zP&r^@FaoXi4{kB zg+5L_|5tB*+S0#Q{q`#6UG~_1<-*y+doMH!N%Yjr5>`TjkuFN({c(+S!8db3rQ^Fg5~w_BMf9;SwwNo?7{^?Xs+j6A`Lxb=;awuf>%&fXFd z^sngW{>0l>&HQqgi+RMU*Fv?<%5hr0OuYqKucpdrl)ZRa(euTjdF{08se25?-u>-1 zKQ1+|-TQgOBIl%7UCA#mC1r)&BWL)1eqmwSx?Kjb(aS2ne!0H2 z+==zqidQBHm#!)wT3{J3D_+9e|D))!!J<_!U)@SQqZoAknDWJAvy@78_pbVSxP4~7 zb$*45?WL<*y_Z|=UY$MN@AC?GslsAr{zq3X&kxCz`|9!4hHVMIvmNtc>O6Fr;o0|a7 zbCsLY1udJFe_qD*(4#3?eV4+{n7u+g=U!jcKDdCrPg{q_xx(x4WRK?)b)~MChkTrG zU=+yr_V~?>=Q89vi)5bZ%dp9QIREouam2$N^B#TmkN>HbzkG&l&2i!F@1@PR+}@@U zwJF&_p-k?FtGAvEaIGsF{S0+TBnX)Il-6tuQM0%^6yXO z4WIOnO*!aO!2|x=aq`-0E4Cl3isM~;)N2K|`0C7j1cPI{b2`+`PQi(~iy1U;Q=tscW0yp1T)qZc4nWtMm1^u+0AMr8~DH772UI zo^|_yjNDm{Cg~?3PxOwbX>R!FbbCdY(J?3YZ4-Czc_`6rq^V9 zE{opCcyQ3cMNjzCvQ3)eoSQD6;}m)6muFgZZ`-ttId3eaCN6BvS)S%~^zz=rGiLpM zyR|pzUC8~n);!1Ff0LCCSIS()ZDSNsel z<~(0U^Q$Veek%6NZeGe=wByymFTTdBwz#bbw_oQjysv82o$DoUQdZxYX#Dh7qdJHA zc21ineT|;7RjChy9!G91SfzHZYPU^R?Zx@)q<0yl>r`1;{_n26CiXmD?UnR2^+{Is zSHAQ_J$mDH{uaZPFKgJ{{LdRDy!*0-*Uk6*trvlpH=EgHRb68{EW|laeVdtt-L<>6 zdCd=xeF9T$1~2CdM42U?2skOK`9u2sPwwb*vWM&LpP2jUYjfB;-{!=p_YB^Lyy7nmhMr&OCi;Zt&;BY3~m2%6oRtpl#-9$tvUGs~gRX!>1nl*if+9al%Yn zUw^$7(r2zJr5sk)V_A3X?5;8?ai;i#cYI%bc_~?1qkLh8vp&;1?bPjvJ-uZsnff!1H0%!lziO%34DE$J z@!!-Jo{`{oZT@MKpL*%$SGn~Ef6pyhcG-rD+ub{L?z1=R9h&r{Gt_OF8LVtym>WcE zy)HYVp~LfL&(mjjPW-dKI)mrNn%A{=dkzR6n%P)0@yXoJw_ECSKce0}BZ5zU_RN0F>!z|PX75EaiLeWAl4Bz!q-__)Z;0L+E6ub0*#rLqiHrI>&fL;# zPWkd>Ubrm(8S(Oe`{W+nH`}mbj(tY#T^5OJS9w3@#oFq;z9u@oLx0Wc`!lDd#y-7R z^~c?Jb$kBZEm}IUZ_jRdbZ5`ZuvZbYGrz78UH{%)q4nWI(K(N0`M+zGNS_H6OSmw3 z(v7gj?+iO;>A!qr>a*z2t{cleQ{L}Me7ooUp3Appeq;$1JW{v!(X#%jcfNMrZgc+< zniX#Iu<@sovEE$ikH5QYf6cG`u(DJ);qRX5dk=Q(`&`1s?oux=aWh!%;KzqiWxBqJ zDn>;%f1WWcFOdB1z~A%X+2tyRbm{K-%0H_PR3BE3tJyV4&D=(*M?I`a-rp=g!2WADA;s-i~eF%fQ)t>O8xvT)Ve$_Os~~ziQ4F&vuvd`1JL%WlzDz>xUQqdUnf?YwnZhO9ITVS47Op zzqDP?qGoc}@>z>+ocMDpcgh}PiI=wb)vq_ZPdOK_$ti97=S*g}SJOJ4!g*`5z30kx z95X2iJ!)N~Bq{gs+ppK6`mrDT=6KkhSIXaX`ln#tk5ZM@IZH2cewrg!=(EiF-SRFr zCRgKk%Y$uhZT|JA=~#v7t^do4e%4==v*4@xk;mM}IQN~N)}Mfq%CB;@2|HH5sg2Pz zm+tX+{K0#lyhP!I_D$=a^4gRz`0uPq*rD*?RPCwnZSGH+uUtKFYk_+ERKNTS!GB)O zNxTsJ>6HGx4<&q;ulL?(Q$QLVJ>a3fb={TMxjJ<{mjy*6dd__7 z)?OPm{i%oBJ)hOrQrRUNH=pfX7j_}D!fxt9eQ_(^^JeP5{`lR*!ORF!XDTpar@x!=*+5zyCQEGO!Wz0CbZ+H zo0`GQgEwWgFGsAMc{wIyMMd7_?QKWuRIFb=nw%Bqd3jfsqr$>nSy?Mw!iuaNQ$8R3 z#5wQnf$1^R=TvP@xZtv4Yr>{Jrt;$-_lWGMw-oysQKJ(n^Qd!uik|pA)AKVkc>^E# zC4Zf_=INni(=9joHK(|KFcE8>$og)}62qIxsh{-c+N;hxp3Sv3bCW5zmFy(J#D3Sl zh_e@$?QHeu|99l@<~c7~9#^;=PMjov-Fw5eMF;-(oi$Mocn~*dWkt98wk(rK{U?jQ zmuQLo+@snVVJ+G%(LE(X>iVNyXPe?qZ=GE=z2so~ncB-P?v=ay&!u&rQQoBLv$uO$ zSL?%xyu!^A-VYR2r+xU)eAG(n-mc(_ocA6cTb{Mv>Q2S~S7k51D!%hAzqN1SEy>9y zPu_gLeD&Mnd0X!9S@$w)b?}mG-R=Il8#;g7)zN;NRUCNa{j}T5%^n`P?RNU@<(G}Q z_NTmd-?9-d-!t)G#{-!+n**z|F7RBoT{UABffQkR*lABe5(+xuk&tB`cI;-L9v3Tpob#;sQSKB^jJG?&BGry$G zeVW>jt67oLd8b&MxfEQhzu5mqj_i{2{J&bxUUR%1@kn*nz9@D{ljl1&Ir7XZj$PZ# zGs`>Dburu74YS-%&(GDl_+oF|zO&5APIFt@5AK=zdV9$$mc*Hp^FH=oJp9+N-%mpQ zOmy0GY2AyZvxL{AvMIlxo4Mv(iO7Dx<)51#Pki5fsp@HPzh`|<%2rcpm9^z(cqip= z_scK*zoT@On6%#Q3IV6-PFD<`qzHYsD9uvN3Nc{eQ@*wDp4@CM=KM8Z6XscFh|4c? zU$i})KV|Cf;PWS^r{%48&0iyzzuxsYbMVZ=1@GI>$z0E|RuW%iImgO*p9!=1!P#pr z7hX~hmR#h0Gh# zJc=_7t8VTa{kmt(Vz>Fk`Vz2)bDjz!aMgk=;r3+8$6Toh_`wxq5n>kGR} zg~v}m1=Cz!&#*9tm6?k=gQkc&Tw1xZoWb+?s;O72j~%%D`IT+*23PhIZ)_KDZf)t0 zytL_H$AO6G_tRc$cQstA*$Se_-Lkq%!rRI|bE>k`9(%v;bJe?-R`=QQY|(uysbkW-cJ}I4z7yIz7JAjJG6~yOZXGwX9xuU^W%gsKv;XXwk8(SoI1i0!ub?%H;nmIo%6lD!qzF~(o*MhDZ#gCaXZMQEcKAAFWS=P+ATFt6bAEK+? zKT8a}uu!@3vV?R0#y7uK9&r7_e)sp^m32poBb5~|-FcU4w9blLQh^%oBQE)ylxS9H~0y9-Art`AC#tf#crC%%J_%Vc-DuD(v0kza-Txr)uVjwkC(y!d>gY{+k5{=Es-(Av@=Q}TYp zG$uRFd(VuP2ppVtvHZ$}4c49KW;p%OeaxAkymP`M`#&pW-Bd1Kec2zGd{F)4^0rC; z+~zo{G?)KSIV^wsV~0YgcrvFwSEC5aVx4<}DuvJUrk65kCu}`f(YJg}!_VA40`?KIj?GKN(FxQW}tyP`PwA;KN zZC+|pR5YLE`|*GomQR0XwHVqe%j?d!cDGyAy=QHh&eDkwyNo1e_J^e(nz~iwgQ`T& z7xNQ(<=@^dYIf>TeUq?#(r%u6A0K#eT1M{?H{ne&Fq+0>f6B(cZMWcgY*4V^*w9;N!EE8Mawd#_;_D*GKy5c_=3l+Eyy6ue)`<@jtiz2uBYgQw_0C2fA{EJ z`6XK41ac&%a4pN~yXyY_KXc3erA^&E_k2&Tu=9CeVDGiRD*x%%kgKPoo1LW&GJXHs zy=hJ5gnL(m_6Ui}9bVybH^JqS$gPO$3rm}~_1Z?QESNEUTlC|H*4*bS>N^h>@bl}g zDz^VIi8E?-=ncNZxdHp5THk(Fp0xhMw7`?kA}*dS=is^bao-~Cdn)bQUT~bxF?ggp zm1l++pN&%bsn|Hz3rA$18m)89Rt?adEXlE(zhU9|4IM_YTLMqNn_(ZxAzZLUw&t^q zy3N#9_a8l}4t}BTe5cNR_g=r|Ee;9KJlt{IKVywVPSuAHu`1(tUpmqvE5GHN zEMrgm_hNTOhj`V_zJ1-t%-r;@e7W;wVTSnid1vN|W{Urgc(YKdGjnTrlh4jcH;z5> z6UhB??m~ytOgrv8p0115Nk=+)&IP`neQPCm%U91nhvx=|1T!{Yo!F}WZ03$M4hbcG z<-)U%wA*LyNc%W7^P+aJ#GOcm|L0qa`?>NL3dCDRS*cDuB%v=Ev1anI4>HeG#k(F) zjNS0*>5LWLjp^P^r|$_}@i{#&*p@Yph3C%lt#LY4MJJ{^FML;O!8G&Wi3Lrwr)svH zID9JqMB)9!iVZDm{+vii**E3?m55oM#&d-m51wdQ>5(4PbnzwTrc5k z=~B{E@%dRqXNCUSh*#_vIr}pvO_Be*%zl68N%6LoadYN<;COjsZ`DF`x%!^3N0fJ+ zd2+KRv0c#9;&Ux)_KI!y!f#yUxV`m!UaECZhUDzDwo816dyQ?@2c5miSrI1o-Z0-F zKDB+JP7U+_dGiZZC)Zv+zoYm(kG-@0xr#qLdo3P+G`@0W=h^cYKHAi^#qIg@L()z0 z#M|^0na#83e>~IwF8xKaZC%@`-;dK@Jh7=eHaYUd*|IHxYu-tC&3OCbhRp58_SuRF36W1PD<&T5?JyB(p1AaPXpidrg5|1nmz`^F-#^L2y3$1a zZlcTlT~_huw=J6~@>yf~_6q&&rT-rY=6m0{w%1PV?YBU`q_wOwr<|+w=6aZ;vpDAb zH}zJ5WIGj~h*L`hrUkBh7O;5ZgYL?Cb^~e}RKmw>GMlg>lSRJ#$7Ud)CV}5;G&MdAb9XUwt@g=XTIm-JMM~ zV{4SrSJin@C6;rHe+bPuA>(GVwc_@Fx7=g7|6iMme%$u|>nyIQ((v8pKb68wmlP{= z&z4!2m3=ozN@dCBS9cBzlmo19L{%{Rqkc>fqUP+MNL?5b?O}N(k?E( z_nR}77BnfR2G?-yT%~QW@EsS6+KIaw*Crjh`r@e#Q)_oa%^%6qiUf;!9xCToH2e84 z($w+U*wHTXcX{Qqik;fie(SJoeAd3^c*yy+I?7xUt8afJ; zBA%3r*CzVEe#)M)!Nz#umtV~>>wGl!ep<9dTlkblX@l+E{*5lr1SJdWWHw}(&CX8= zKRaPY+%X36EH%-WD&NaeKNd_n8*Z-hb=&*L>nEJ~UNc|%&WGAPQe`zY^6T5Y zX7jI!tWljc|3?4MiH$Rqeh4+rxDzfmH~ev*M#22d>|a%i`m2j11iLz!`g@LE`MK;H zn{u*Yomso_HSSUc(e01LV^i0!xZGp@%Z-_{ zxqQz>^Wy(sY&*Ip9Y}2X{W$#H{KobemGg-eZ_+Ey-;3!s+4DT*-BXi2>C5K-l;L~- zGvu?)m&@xs=I+xCa?Ustmb_9pTBhyxQ-R*J{9vO@J3H~EbOzV{EO7zFA# zB_GZ4lIU_>#r-_u%;96&pB9_XQ#`A|a`NP%pc5}kd!!;~@x||(^<3!G^>npIY@eq- zmAl`PQfl^Z|0j-%E|`0#3=I_w6!Md@xCq^VXkclAxp^wrI`7&4&mzBBjZS~(HsQ6L zYV~!uv!k5C5~fCzl289G{K`3Rn3L&Iy!PGEwqmp7>{FX`PEXP~`EUAS`~MGK+HY-- z|Nm?L{=f6@e^~m%!0ylW{eQpd@Be3Rw=YX>!)yKj0=1b9-}irbKL7u(?0--9|NFb$ z{_oYlzrO!hGtq4Mvij$z)4O9DUcF~F-*?4g;{RA?Z$qCGad-X(w>LhE&scE(|F_%o z{}t@7`S*4E{{PSJ8-HZn-}mLCef@9wbKk$%*Z+NMU-$d;eUlGU%nHPBGwlCwxjejE z@x8%K;~7Hn`<87xyyQ^As%lZb$re8KmhpQ#mro6UX0YFQhM&pvx-;4vD_ORp9m(Q+b*ScJ&Yc@Jn9KE*6^t(icXR+mnCkGNd zYNGN_%4lDzntErgu0(LH>h+vi#vhkID}J5AcRBv<>`7AG8<+3Mm=t_^wZ*bpi)ty) z&yHu_w>mGG{yf6M=X;Wh;m3J5J!a;1K9PHo>OHll>T5}s)4tqE>fesuTU)ujA#!f? zlx!y}*Er3uS!cCe7u^gG-V>xWtJwb8RIB!v`tolU8fo6Gle36BfspkDl^Z`p47c^dT{xAu_RN=8=cn6{T@1}T1`^=ck@R0%)TW- zn=5;sJ$NCJ`^<2b^U~xg2miDlJfidQ&&YE>i)|#Jxb;+>2ZTU^Hic){?-v<-U z&wBGKCT?Z={=*#3yAP~buP&FkVRGuBgdJ@S2hV-{Ao{7~$D<7v$5zR-@!Xd3tGZFA ze0aJ^nAWw3#&0Q^59e5KS2E}1Vbekd-dB^(tDS8Pu>h;k?gPvpGcU$;O?g?sqZ;8oi|g#(=Nd9a2b6Kj~0rXVhdC_nt{7fS1kY8Cq+jOPThObG3raH$Ergz(n;|PqYl|D^5t=JTe3@{`|@O) zo{r*oF##t}EZDeer{_smL6g%Kf{QY43LR?6V8}hq+$P^3xp1wp+p$RpoZO7mmnW>L zdtoWJ)OEp%=L!9;zi$7ITcav_P^HcNhSG-2UzK{D8$FJHf5bn_RfJ=r()1?PPs%|i z%n7SHQg6MPcld*bKyF!MyL;myYq{lqwJcpt_Dd(QH!4V}ignx0yfa5^?@ne<^Sy6H zLm#;>Sg|GNk>i~i7Z1DUv~x+S4AxL>B7&k^)968pj;$S7mA zDYoF*tm$nBGVC;;8M{wVH?Az7-(>T)F;Kqe`Z}p8Gv~ZEamo?r`5s|2x3c%(35NjX zeSHhOuJ3sE@QwfBFKs`Y4^K#(Rb_ZQE5+yg!W(%K+k#)*zwanot0QsHSmKiD%{gu# z45##nPF!n|Y#e+0&cVa`WSvS67uYbG%`;zlP?tw?0=twTn^Ms;)_JEKi|pe~cP2kG zI>H?OKc*ya>-5tK7j`!Cf)$o7LGR zY10drT@t>V`eMcBu64CC+P@Tzeac~WlL*}^_xaPKyziTIuJ{T*d?lXmRowDW^p~Pt zpZkGJqR%copSRwi{hafsP0RL$O&8f#a{ifEuV{Ir?W`*gvZf#GnCV=VcER%#miJBh%KSwNK=bivH`E zE6Yvw?$6oD7`$`c$}3O3uI^pGyX4ygo^9%VuV;znWEF@CpDXsZklMRveRfEek>1B= zrDs`%<=4en&O5-HFke-!@Tr%0kbvpWu~U7ow;569IjI?q4VZSpA$ zd&u=tP-@zOmUHiPm;K3`uxDOeVRr4|54mqnF6BJ@;%35+y9zHahdqe7XK-kXit?Ie zRTmEDrLlgslhxRNL@J2se#5)`7bn)HPO>v9`pkFuufTh5%ibL>`xAF8TXJ}g&4vg3 zHgilFUCHHQ#p;)BW<(O;JwbP>3 zO>9ozuw3A*GP{5LIuEm)#EK`n4zW9HjzvtZxKV5H+;81!pJ$gtx$_v)jOMR1P>$R< z@8F5w*;3j^GkT|mRJ4e*#o26ARaZvuspK;>cSDCiw>y~{vwNcEw zN=UOM>^!fAK9^RdOpbDK{KAw^mgj^c@7$R@H=5b{-j20Vh6UDt zyALSqb8+dh?soN8vo^n;qIKBEFIVsGo!FTs-o4AO2~^Y^?w!AK&cw1;7PfcFJNZpM zS58cOxo?Ke5nsng26N{5m^@E=w=O1QO+gm#!|4|qg(T!2s!uGRBO$l%w(&`RdG8CC zJSTsNhT8mj8wS@P@C z`~(jZ3pv9u6|rN*@07~T1<#t!eqMIcWLo|*(`XM9o92l#*O`Q!HnW|2jO*&M_Gx>= z)BCS>v~LXa=9zd@D%OmD{iV#kL8kocz4*NH|7Pm^6*^)+F~7>=W_Hz;$pz1t8=~K( zXrAX5VD5bXS}C>pVWhdr%+#1kIRa1Pa9{i_qxqq@ zLW-u@qD?I%zu-0e?P3;@w)v7OUfJJ-wR#rHI9kb{}q63bW6;$^YN(Z4)yu; z!-Rc-Kff_#r(1F4tH(VzmZwX-^Z(!d&t}@yYYj=a1dMZ^u!(=*{<>}Tt}sE{mTAR> ztK2H$XXhE%h}-V?o@bu$t9rY*X`U&U)#1jr{Gd%qNuKJKGG{|q?>xxS>fqcv>wK@w zLV*c=Z_jURadPOAZT@+B`PsPg57oa{b@OH2@1D2d(EtC_ZRePL4mmII$m{E##0nPs z2Xe*9^4}kM?k_(1v*cFmll`ZqZlvVjd*G%2hok(JVyn_b!EaAAF5C$Fs~A}C#g#7@k0 zv){X9x$p(ev2T+3jBT!7)7dJNDZhK-uH6p>!&m1mJt(E^IM3}j%jHc6l-I}q_F8_m z)W0R=OYq`PU#i5zZmqNvx%9s z54wL8Uri0niJQs$>yELf+jE0ESC^+guec*GqqdfZv-X{G%f*{J&rKG2`Q~{=O6Zy; zkuA(8{J+ZZM+XP_>PKx}E?m~ESareoNW}U%4_E(8SsSkRM0}ot(krKL2YcpAop!A{ zS}d_IMwipZ=+wn9nQIH$znogJ^4eFg!n)Q(-5EBQjZ1wy_IN!%v$vJ&j%3T#>w8$F zPFfe+az9zUE!gn$4xUE}UuC?*UJ0F4o>+MJ^MtCCF9kPt3Z(~BFhAr`*($Wz*Y~Zq zhEAXT-=tqQocj~(b?ce-z3;aYvDGTvc6Q;8gqErG)*|L|4+9o#VBfw)`pjY8oF@kS zifu;DJm$9Yr~3aI{MGxgKKjGtv-Vks;zK0V{;+<|Idp%|Mc;~jeDXd&@;|1{{4m+l zKIicLHMMfu+lnN2R&*2cuowcoL3PH(-L&7G`siKhQ9 zUbmfi&x6OUC$zx$;J4`>M|rdlJ5K+3C1cH*ZwlYFZlwyowrPj#;32WwAV10>&wo?Nt#6xc49g!CYCMuEUD_PyCv!EL7#iG*RPvhyu9@L(GdHT zrxQ4D%{#Y@S8Z($BeQ?+>eT*&a#7#TB(TUu+<&^IcG}?u^QP{!&lGsQ=1;vXkNbwX zd$jE)%xj7$s+vE?X5TB12q~7&ZGzLSJIqqq4kvG5R4*{Tcp=jCU9P!++=4z21OK}A ziJB!-k4Sq)CTZAC6uk9??fXn z&H8Yq$83ksevAmOddPP8^Ur&cUDB5R(+iYKb{zb`s-O2Y`%c1+r}4L5UEguApeQS{ zurhhgwCjv}eY>}_y(*n*{IX&~lf|i-C3@`h!z?eeDR1J=j9UEcZJXfC=Egf)Z!Fpx zbwaP?tn6i>s~-h#_I(T3xZdbft_I)a32!3!Z4R}$F(vWcRgv7e(suqOWg&r1xe(?k z?ZD1?x87DNG8$gZxa3n-@1L?DH<_LLbiIbzdvJ^SVNvXqLsO_~?!y>_zzSy1i`CT*s)i3FqWZZf5?xVz=mp=F`weIR_McCm!tBIN^!a zsgFIqh(^TeZ^(h9MKBh+53h-7{Mc4|IR&7!~UpPhkWpZi^w^>QqUX1aCQA4yx;J8@~mxx=w>%mTBkwXbdJ*E#ycCs@aP$=;(0D|_!0sut&!8|ojq z>}S^fuWucsC8`k^edZv7FDpf%5b>I?l7x^v_6 zAMspe52IPzb6bq`>iwh}qZfJv^(>stwd%pQlUDO!$FfTGj|n{uFPQKnViBwK zF|M1lt`{td*>*!F@WJtm1@1}>tt+E0J273!*|hVa;9K=mS)2>?Gkr2k_uWZw$?hzF zdXw$5IB2vx@}=wNuH{~$;esnJNG(ihPLnto^S#4usmc1VGmDiDamXbmHK!F#sInKG zVz4Zc<@hw6kCH!q4@+J??!??R@!SIaqaXSuI(A4@?#{NFKJlK(T~lWNs@>UaM^`Y) zoMgOxhSewjYQnLI6?dhtCa|$Tl)GA7TzFm}Gk@wv_mb+vvT+t~na`z)WYjh|s+;$@ z?+{#KoR(h`TXCU5DQwy7ot}NaH$3c={BmUb_GS9w_gH7#o8`!VQ_1v8&ZWtRi~4&u ze!3ld*<4f4f7$MBA173|Y1~UbKIsKVyNgY}9P`efq0g7T_WrVQhO5`FjV*R!n>{?! z6@#XyO`NoEVz;xPcyppvsYr(5jDB0ifM|sq3Ab+tI`waAdgpfYK;X#>?o8Uh-3r$1 zdicjgi~B?8R3ipS@XmGiqmNHVo?;03bJ>`e=l6&8a%!Oc@4lb@n6MpK-YyB>=P#F5 zQWuo*;Kt-No|UsK9{f=0_b+Zx=?ap2cD7FABLAn!8eWw%wmUH&f0tr-?(vbi8pkX_ zo9s1?RnFQjeAMFka>sqbM=kt6*DY&Ilvh7`xaEiWfsLN?<{xAD@x?&-TPJ9jeH2SP z%o_J!J#!}fn6_sI=*)r%e=hg@SpKPM$w&^3H=bs_LJs*UYq>IoaHP_N1MK?z*dQg~jTwzO`)qw9s6G ztjMco)1tzbdcTSdTbloE^%}4H7UhvsBbMo|5;$TPc6wFG)WYeZk_O?CQ`d#P+q8-oe*<=%UiD>f>GaT-O@-4# zUl#8_z3RxqEv%w*4lIUp&1PNQq!rtGH#KzEihHS{zh>P_4P`qU9yxViTsepUv0}<2 zr*4c1hZ2TsSFbs>;CXD=($`aWuQ}BenHsuFZOxrMlW`r zT4iRF6*=|Xlhtdom@l4M#gnyelUCuMo2jAqH_cyWKk-7xXLngHBNGwj`8iYV?iuV! zY?<0`{>!cYIB$P}#-CH#cM~ciMd$GGDd)w%zP)eP%byWpp=BTRqd#=xZXeD1&<$xH z1)g4261n&EDzlpvdaEzZte6gtnyGddV#Br`T#*|3>)6iIUHOY%&wXq`to(@ z^iat!F{z=y_PpDqHP^m85^U_H>wooD8y(vP3Zvrbp)ZZCP79=F_TGy?aBKIcQFjPof-7kOZPQngBw@>C#rFjaACRQ%o+A_KJ zaL2FtKkC{tnje}a*6csJvsYE=M9TZ#ax-(AJ=zI%`;YE?D42Wdetx0UpVb{M^9`EP zY}T9fc{D#03tw*6e6}LF#AT=14E}Ai-XGsobGJ{{-~Rs`!RpBiq&J$MSuD7k`&{mk zF4qmcH{KhjNp3tcbN<$Z?UkR+XCFTW5bccHin3(^!L!{ja;gp&8ZP z_M5G$S#af#_tGz<-QFKo{_ui5#is4=FUH7ebH6pGKXGj>bc-`MpPq6tNh8S|hn&j^XW}&%%4$riVUh za93OXXpIBk4-I?6V~29zYS`cW@Zm~X$}c0EO&(_J4*kk~_eV$e=7EEA#3nUbN`1`W znACWDW69wT=~xwJb#LVbE@o$vS7zKb+P3(0+r!A$9ghqoBwrq!DC~RE!-GxkJjY8J zmnFxahOL<@F>zjD<(kcwirOVMh5HYF*eLKa(LgDS>E%rKm3HfQO1NUmSG&YN-VX72M0|DRl3I*lpMefm9>M|Qh-W|V&l2|oRK-IJBPy|Zk#9$axU zSemE&dS>^zCEUrcSo7~GMQxe*=cMdf_Mh?gxyPISy-2JUOqAJH9zEsw;js4`3+1=P zr=7LQ*?)9a=?%Vne?CN37q?GeGtq6X{_WgJ2et3JU1pSK+1J2*{N5yYv$?`NZgW)m zeuipH7MgRfFG}ikyZXu73HNpeOUo6?tiPjK8+G~io0~FPi{>)ieznVeRhThMzX2%m2X`&Y^Qc0D2y zaQ*1Tg6q7#x)MES)_c`xzy zt^Tr4)5zo8``Jpd_dZ&Me?Ke3wX;%pzRKs!-s4k2+wwz2PR}@VP}&-_E1&h)y^Pbz z0X-~hE>3fg|J<|1y3IYT?rW*)(#tmUo;|9lI26pG9~6I2hI=jVr_AJzY$cPuFP1-= zc-?x-f~O%iP0w1gwev6cJm3AQa*3_&{i|;4Wt!zwK5VVX5}g?9np`Mx&n9_`uExP5 zmoKNSdL!Y;*Rc3-KWUzF=Afv&18)4=cNa%Uic8BSKED2xJ;y+PS4od|i}at< z+kQr!pKo$5UyJGdbzsF%D(``l9RFS(K3q7@s~@XpddeQUwb zO>4|0u5P+_EJ!!I`Pu1BX3zS*`?~j>e~{R#UGH^0(Y5y z?7yNu&A{kAXIhQP^)I_lT%4ZtWlD0!lT`m-Hdk){yT(67TWneOG;6VC`YflUcb$GV zv8k`9ajw;$N!+|o-F{2N-aEc~`(piN`{q@Azt-=Q7t40JV!O%wY`t4r3w}qKJ+*Z# z^I9Rs?H)Jfw3y|KRR=t$wQ{R8Wwd0UIH%F@AXt2^_@S(WnLVcW?BCv9V%e3+IawmI z^zhdUu97!H=N`5(jgz?Q$~JwXmFQm0#GP$dKV4d|e6Me9a@^b8SJFCf@`K~2KMTAw zp?1@=)>EvLHf8mi&N|oYV_qA6V9PT3StjB;(v)9plszuPb~^56n)p`k&<7LG-&>^D zs43RRF8j2oZj#~MK7D`bZ|qj}_TN6e`|4Hpuy#_W_vCH%2`#bRZ=)tk79Kt^J86zY z#*D2G1-Dsj39vqH#k^$`_jiGhTKbp1u9Ny}-TySL+;-O~*B7r2?LFvm?Z@N!@-}k? zC*4cjv0LQSmY-%46I;aYerYR{n7HKnGV?8m3;H?NzInj=_KVHdP00p}{A1lVzrW;L zb>TFh+-1JQOP~D>dFR*f@#OVovrQis$V$GE>wfm``P$hV;|}gj%#_fUlbO{fp3f`l zak2NtlfL%DKdh1`A1k_Op&vfmmw#97;ZHXcBML(sW=BMAa?zQPQSwP!=OTw>>>2w5 zBKjXXHlEnRXQ)=Jsr8HNS@F&vz5G#?ztbi)>+BJB`L=TLL;-En7J<&H#hp7H4TZk8 zZA|NFb=k|{d`2R>foqfaeQgZH|i)YR)Vy}I0@yxkd;*&%!oH;k) z(8C!&gXeQS=RN!8!IXCfHknL&1*(lyOHDR&FH7i&E6XU!d(m_A^tt&uB40I3LX1Km zo;aACky0mHwRw8dTCo`O`F^I;UHR_4vfX#s=F2a+y~o#XJoL&oFK13mkSd?Do?8LG z#9~!jO~qQ%*?%O@zPYgZ*n{Fz9@e?bzNmP(m7kPZy@su(E#rWxH17`{!QEE(BL8Hr zW}L5hcC)<2;m%y?8Q(KE>z*e& z)SSMZFtOr#K}@;f z;u+_;w#PoHo_YCH!S=(JBIWlBZijwe8Tq4^e^%}8Vs-9t!QMS5e;X{nmk}k?W-jky z?85eW&(#h6S|L82qi*Vjs zUHe|-vuNzzedPk@=G=2Tb(HTA%jdu6F8Xcwc5b5K%8zY!_7rZ<(@j4x$$$6XwufI1tahB%Y$|q5IFj{P66+;FHZQ6|1*;^W?$WT;aL%XR;rg)%R#o=H6ACY-+?>lzA0p^hJIa^{ML3(aZkTCg}R* zXYP*GHGMDc$MvT^Uw@-ls;bRR&G+%e1vhle+&kDVrRv&bG`IeW?Y0%I{NN+sapskg z^0aFaJ5s(FMpU@j2&4CQx*M@-JT(dof+GjrYC9YgT2QmU5>beeNdPJU5?9 zX&0QSS$fU>U25O`8JONkN)dB`s&LPi`VksH&%qqo%8%-OQ&hryU($L-DyP= z+^qM@wp|YXtJ8KbaffZt@0{n`FMnS7@w`F&E%Q@P58TmA>`3_QQRi-RL%As7#H0wf z!pmB$iJm&nq3sJFCrPe~Ir2qjM)A{q2j(@z9hz*L(({+0_}0elE}u^CP+aqMh6sP; zqU|@E19Hq;W*XHxWRTc_U1Vhb-$ z^h#PAroUPF65HdOU&RkCf9>SDRDHLJaS9Kk^N%aHWn8|PTq%B~WaYsW_H1v@D*g|1 zk_z|EPM*joov$ugEfV>-@!*H2PS<|y+8wx*J@WCq!yW4!Ov0>t*JDzpR7N=Zbj;p%p-r0Dt!1?g+sVa;7 zWsg=HE2bV+7V4QT;Vt&#V&UwhiW$4*j(0t+)Q+Crcjna6Y&oy2>&td0q$G9B)-5fU z_;}*shrT=p#X#lCO>zc{B))v=+t%oE>;G-*TOT^U`Tng9xNGoe?vEW#s<$KfGyl(f z`0$wIt-0s86`JQwZ0W3iovU+E$n+e$h>l#+b0d?a;I>ltlr^CfFHhQSw4TSZ`H5I~ z+Z{ILvZeD&-LFN+N+oE_mUeTgc^$2|viSJ9ST4_4u02yu9Iw*aExXsGXXoRyO*gtt zOKl=A_zBFrFyTVv9+%>a6TjKtacWL5k(akDVLPaLPA|Z^_{d^54a-$$RhCaWEL3Fj zGPA(#m1e>ojYmZbw(b4uf5)=vuu#so1J3r-v_8rGJX~vK>Ys8&;kCVD^!*v3-&B_A z&b6MKzGA_f^Vf@2T@HsxX>ZomT9x;EwdH-C!sFH5#-2X+qcUb#>35ctHoTc!X}|4Y z!S_jT_8&|5v9;RmxR=?qHPfbkIbqJb>wvORqlZz|+i3|8L@PJlJC`cnsoJ|((0|k7 z=Qgjndm}|HY-QYR{`*Y|Ofvj<;N?RDrHw3mUl(uxW;m~Y{?0V%o{E3A>tid*JjPurHj?Itba?|rfl6;WOcsoyW+Z-*vbhF{L2Dfbs0R^SGhCIQ#;qO z@v%?J{zR7;!D)Q$l^M78t>KqFU-78vU_rJ0u~N>>2MfA64X3;mzGJY6d3v_k!=4%S$fm=o7*zl1#@>j z@@c>5%KX>HV%G9!6BqxUxN!-mo@%?`TjfNvieQs~Kncs97cU%5`c+EjwsxHD`}d(O zW67g*wqLx5Ozq{Iy@LH$Xm=f-;CAyNpYj{FORowol+AhSS4uJlm;5zQ(K)z4=Xt~@ z!zJYk5!2@-if-7La>Qi)8|m1Z3Hhf!o)DJ57%=&fAoo_6bT_AF+xLQ;sRoapJ~8^; zB(bRZa@`kJLl^bSRefdOtn+t#HS{Z5Hus33?6GFAV^=2rx0G`-I^mcb_x|mTqkm>? z=~n*G+MOk$po=d!&AJ}W+m>8hS^ga3HaPB|CB zl~FI>?hs$Q(5@^HAc|OV!Dhg1Z+A)>Y4moxXPAx`bQ2hmw0QY*Nm@Ji~I= ztvMM>(&N+1y}M^w-&=Mrt9$#(I|r|>n7wg}bpMMyPp-@*UU??=CF|bsJX;x2FSg|6 zrk=dW#3kjg4)v7#6s^Adh(9X)TVZzkh33!OuNPlEdQ`plXu^?$o!#ee?DjOyvx_z` zf3V}H%+jD=Y(9T$@1{LatTl*xYn;NvUQnVLQS833_OZdEg5G?4|EI=WZ=W+3O;*lR z{`{LEaCx88?@5w!a;gV5eUH!;JT19z!FRQJP332jjBW<6JM48l_#yYR;3IZ>?Tg+` z_`L1Kf2IXtRiC(0^HQJ}0i@=ofL6d78WXt`*2vhzhzs=|fw!x2clO==U)A-QY45H3 z@&Zh;=AnPB7b@@z6iG{2O?dac{n`fG>syv;uH8HNY-6$i+1071x9RXlZhBI0`Rg

|M@BZ|4;V6 z$M^p~-CqAE{rl_qf6cQ_F5DS^c6$81r%lWL+1)xeT`K)?{q++?Yf@`m%-_$qDlz%* zD9idtMg5od(x!Eeru$N_A2Gl1Gv4s5zp2GP&6ZtJuiwi*x}5Rm&yhNf+3!3wIHTNU-bM2>`<^T5aX`;)U%f|eBO0UbatL)pU z$5vT0OLl>&rP9Bbe-`G{ZRpqMS5BHguQFk+^P2f=NA@@N@qYSaVkNQmq#$UddsF2G z`+YV&Fa9j~P*DHt{g>x~tydV18vcLxb!vU<7uL6%N>0A7|NE`KepkHR|6i})@BitY zUO9cm?|b!szvl1%EB-v_=lcEseqDe2|J%Q*{7v?jj#XPF{ATShob~O~QUyN%k3O@%4X}>eky&J9@M`W~!slyD0}|#6MJbF*wrP ze!@k0iH?fShK22KbWU>a+?&MpF;Vo&OitCh;G z(khdiz3ast?XOX5?n=fkURSmGnMKvBRjm1*RXp>qJ-l;pzqsGzxYq55Gm`{r*RtNq z=z8JZGj|K?d+ow`!PPq_Ou8AaR=3N(Jnp%PYnJS`)5)Ge(J6k;Hs3G#)M<6PdHR3a zck#jwrORg!$s-Jqf?_T&%xTXDK$Hd7WTAr*@U7?wNQCxhU%kjYPmi9d{ zp+zg(^0jvI*8EXnTKIEcM460@$R+=q?@b=5@G&TU&2eAQ{^Qfu@Xd2wGSpj|pUk)L zjE~M<{4k-dZ}x))W#(lwbnZV$%x;(eb#*;+ZFW2JRIi5iiyeukOg3xhJbvWR`mHe8 zglR>|tL_b}jvC80t+!}+?|Xsvew-D7d{_Dknatxdkf0uQ$1F zD=bvGobkpdw8Qevk>aegPoDGYmuxLL+})eLOzmY&ZEL`ut?y0>pI@rlJ^zWdWBVPQ zx(xRHySgt*RX53e+pN!MpwP}N@#40Us3#ie&#aclgqEIeR_&j z;OuNKo$&8kJ7sm+&t-NzeXv;7PUVgM{PSnexhj{>ns?63I-k8leM^$AYvDst$+JN^ zuPS5q)X!);-*Ky*VaBJ$GiHVS-JDZl*}4*Ha2l8m}|_@>I$rcahv)+4_ZYQy(*4*>pNBoFOY`qlM5x+knEdaKVmcVVmc= zOfC*8aPfF<+W5Kn-w{)R4_9yUZ7*P(*3B&b^6i6p8#m>i`6!xb$@KG^mB}mZ*+Lcz zVqPDebHMF_(e2I7UCTdDZd*JfH0r^Y@`up3VxpFHPE;HZCwfJ^xS1 z*M$G8S9DJMoUV9zHZ!x)-pqN5mu=HzjcP07)KY$(+rx9BrShlao|5xi`+1idZ=6!q z<@4O6VBJiW9SVzI+L%pgkXv%oXtCgiZj)$l=jERz?>!Lu`t+we=oiBH_v`i`S)VcOW#eGsBU+LlXT8w9;^4l&~=r#w9;F80$+`3 z{G^lqM_c1;wsN&Eogn9T`1;<5O6yoQ&*kGdlW}85*Xfc!rox7z{twRFmfr9nZ|gki z3EEehI5r(HdoX`a+p(#OxAt=@7Tx!kvisr`vAkfGz1hUgAb;*1Ntfgg!JnLXqOIp%{?aO8CM9>zT`el{ZOT2r z;!XPJj1PACux~wC%r87Q=j?`;oagjQHf=oI%XIyE3NGECg$u5h<}^W z{`#iFbN7U|74LfZDq~9}|FmqgnI0W~Curg4#ihYq&Wm1Z8qU_4EGOycCf#W;^N@?B zH0OIxBMZH&Urg1Re@=DFxp0lIUwuvLQGUtAiw-CM5Za!$en}_L zHH2a9YV)6po2{+}>C_1&UVnD`qImf9&YWi_GM>a5_tsW?Etxy>tC00G|(pSBc=MH(h#V?8QXgjl5r2OQSsXOlmuM^gP`DilNM&pF(>x6=qX6k(_ zQ~&Y%)WqN@ZsrvmCovQIz=6S!>m%vWg5PpZN|^Cm&f({&i9_ z#d~W?ko{WKU>@U@^95d6>gAnLcI>#Tc6E|OuYtSA z)+e)?RPASma7WrLwo)h0&`l5d{w6xd-J9V7I%pl(H{ zII~^NRc_7RfY~RD6qh>AzL_HF=KJwtUb~2Q?z+g`TP7rZRR1Fut19&4oSF6VlP^P) zHl(*taa*5LTv)ws!h*1LE`LRjH_4`n#Vbu(*T^Pb*sAbL=(9S@*>L5p!molCZl}dxJa+5%rw8{{l;+H;m?QURwp)bYy?C+Mi^sY? z$k_+xXxAxyIMu?3Y>j!F=A%%^VAADn9VeYdqD? ze*b}F*!>Ty@@osVr<^-#(InklcI$S_rR%E?%VnOFsm|AC2zsw=5y81G;ca>%eXuJ>Y{b79R)Gh(oRa8*($j!V+(V1PK(T|&&EB5Zpybb>snRw-WC*# zO{kv5kS}%K*T*66THlOxi(L1L$G*ZpWSq>T{umsaH7VzU-05dGtbG{VlppOl&HDcR zPeHvEH3xsscC%qrwmFiKGQsTQoZ3y#*q_~K`kp`Qerr9anuAfKpUIb)MO*hihJr(wQ6fz z#f(tNUX#Tu)`y+X)lyo%_l5VYCe6iTl-o*DcfLn zZgS9(?E*|3In|sIawl$aJlE7%tFcMNYvb>(mNj=O4{;hRGEPqq{``z3M(*t8l8C?_ zx2Px+`-v-A9{yPPGc#v*@#ppcr;Uwbb0@C1+PM3kXlGmhH$S(6(vs(XVs9p7Ot4IB zYGInbxm+!yQPed-JIY49!_EKs!ZW&NRyvPN=9ShZO?%7noBiSXrP1C|LWeBR^u~OM z*|A}r)$)a!DJGYCvRdYLch%+-I^Cxl^Z0$1y(l{VPW80ARSMg;D(^|Jw2iub>tTN8 z%Y-@aqk9W4E;PTtRLbW1rh7}J-cI;)toW*{hT@#KjW>gDa6aEzh_^1F2{wJwLH7*9#}Zs>o+m1K5fp>jTpq4b*xDL;;E+xWpH_SrBv z|5}mv*(3kCS^t;Bhudz#x7S*a3sE7c~?X6svR+bruQ7QFjWWagyWj{?RXY>O+o<_g!^{_HoJ zE&p-NlwV0EmzoysbJ9wSpZH7b>6D6l;&=MzJmdGf>}q`|s`=^TstwmlBJ6I?7dloO zmw$i4u9%sAjZeB)Mi^^Wgf6s@~Rnl!)@kb9!PVQFy)R7kLHvgXA zo*ui>X%8>IRQ}>K{l}zxZu7MgX9qv`J)Ez@{r2&*Ih^y`9xuPLSXQpsY_8Qgo8p9x zOQh!cIoL9-w_O>2*-68%^*}!N!)1a8Bl5quU*v1tBY9!l#SaJct=@gE;MjTbgGr5Y z#XIKn9P^|jSD8ui-9C9}Zly}kKA}4r4f4{x73|DE>Pu7?m_+U|pvqGqj?eW>WjaMM%Sx5ZAEw^nt6N_v!3bhu3ZVW*A@ zDpscz4_utI`slfi%EQei-SQIs%CBPNe{p0an_TKUki59SOQQd;*YwBfue)z^NN$W@ z7!_o3=w6kXYNa7KE+)l$HBgxu*Z&CBLfpmL;!0=)X+n_wb0?bCOT?K<1O~otZ7t z`q#g*q-MK&SZ_3*w<*V4Yt4H*W{a;=8xrSAa&F&yGA(VbPU3wJsg+FSXXf*K0E57Cy#zD)30!2j5D{%nsK+E3=>Nb&3`>rP!VW7Qtn#Chg7#lL@^_;&7b z@4Urxr(Day8Y&mU3tkgll;Eb+rxGAB5M!YonJWbl>3c)F2c<_Js-|2>2Xt$RcJMcyY^(_5`kAN zR=-3hIdRN!s}MO{WcK>4i2LWmnMb;}xVig%y7ouJie2lr)t@tN3Cmu;6$s^1d=U8N zBKv0jOCJ}yeO}YBd&7?$r&&%``<@5Svb1iwQnj())2V35qw|Zsk93+Pso!!BUl{w! z^6YZ)qQ9O~>l+S-A9yVOu;5ZkW%J>z1Mk?{uXrGC9MdqeG2NU9;ak_ODw1r}LOj`75ood|ufVm62;+Sx;)5 z6>Rj|X=q&S#P!E$`$># zOFykLGcVy|$?sb!ASBu8+w)U9)%DNIqDJ>X<45Nugy)KIOTMf%y(OF;E_3{mZg9~* zkyO4z87Hk53tCJIar1b7ZPLlj92VP>x-<=nzpssc)EctVxF{f=>&NP}ClAVk3K+bS z43*ix9+v;7b?@Bd{aXG9mY+|oYnjU% zx7^ybHKst<#ZURrj9nS-3%u_v?c2iY+Pj*kWX%ZR?dSWUsoKb|Kxe ziXrK`&P&I(4No3xuAC#gssDQHs(aUK*{k@wf1VPIG1<&1@%BMo@uIZ9yzKAHj~7eD za)Cr!Utekte`h>-;=Y)PSIXrjJIv-;PFgzG##mCawp8IuvG383?{A%ATF|YfXQ93L z;c`(+d)| zRR$8OsgN`3 z=!PSz8V|m6_+I{6&iLoty_hm5+md~TQp;*qc`us#DCdGzQqa+f9~gz=3;p9;j`-dd zmR1Q{e9p{c%`w5e2@&<_1Rtp zJGLA-*W50t-xdlVJt{-5C>&%gKfb%YH=S>7AX|!?-OGpHey-v#yUcMTX3q=n18H}b z`OT?1-GAUn<=#;9ietMzZ{+sriB2kC)4HSR?fcuYe@{Bq-+w2y@bC4v>*PxKF3z|Z z_==nB!85zX6Thqzunt)3+jyd}RqyPIzcbY@Kd?CXINn#$rF2!R-qrGe6@F7UX^Kxw zNq1#kQO%vIJKa29zAE?i+Z~2q#KN*P7SDYbmZj00eo1og-0F~+?T5~vxh1&b!qlmD zUhlc}S<-Z8t(EQTC|uW`!F8VD#q$fPvEET!Zc0k*cdG0ca=S@+oSkVc;=lNLaBj1# z(P^8-f{RyuIP^T>oWh+mdLrINLV7pVCtW?>ntZoCgd9Ft8p z#W==C&NrLY9>rq(da~Y1%YFIfp`52ToZh9GaH3QC(aotl3REx7=v-JLFtPDjgf&wH z^I6mRXEm1`K2j((QBo^T{6LlU5&^5_57|UzMHX&o?p6*^IV!ncu*8+U;oj!e(|$ZK z+WI`Ft9YU6)8hx6^=_Li`d`WuXtanaX44Y(i~k!rC64b}^60haCLO)MY2QPfT@w`$wrk10F7PKGJZ`lF>9eRV_9{V>g&y)B2Ilueyvdc$#A zs+;n{>^SpTq5WsdYVT#_#d4N-cN9Nqz3Ogyh{4TFX|c9Sl3ZuvYQcheVf!kZ=WB-? zNO&%*wc%pH1fN}LYK`}A2wL)CIG-RqltbX@4{_oABDitm#g0{i*In0LG^mkM(J?YOp9AeFJGVie5_C7MXBq9SKf_Haci_W7yoAYw<~Do z_RD8&Jl_3?j^*e3ays+J3W20cGwhyjwJU7@yKwHCbsq%hJ{MdpJ;O<${KgLcb;}Ms zo>#5bdx@2KscPCKjm7JqguiHabe@rYr~K7#wZ~p1HZM)v;skzP^tEW-wEC^GukhQB zmx`OK>|Wlz^)RnN@8`tS@WRwL=gV9?nw~@+`=}&xG<>yw3jeH^>7}&+ZW>*^bKTbD z8*daY+HfvAruk_^_^pUq&%ns0gGvW^Is40ftUJBGZ@Y9%Dpt{LrD;B=P0{jm%XcKb zuv)X;zkOA$b?pA{tJdCAy#D8;ZlqS;l|?JE{%q1;VPoExY|1BPDt*f3N7e~DqvF0* zMWMZQ?n&ob?_Fi*eauunWx|RqmLEF~tUG?9_3!0Gp9gt4iyw9f{Hu3Uez)HC;+)u% zM@*We1+NM)=ex#eJZ8OOV*Z6aX~*%ByHCU)%Gi}nPQ1m}c)6gju>5RB$`t)~PrNrQ z;r?m%*n}xhs!+dnru47qedQMJq4)VN9E;i6vFevz!k%hxbx!`n4abvI1dVnn{WY7+ zk-PX&qJ-v}%{`U+dAG$*N3*~C&lFF%Y1(Q z_Evgs(Bjq8!hVI+vft;w^<`oIisq?nATxvQ*5De^6HklgckFk;mg)r{+E6Ap!el$)qQzVhB*C$0n%*Oba)4s0Fk~3fLJ!^fdx8)$yt)e}ml5Z!7ow%K0 z@yumXXQAuviyv}2bk{uyO)lcl**5W2(c*_YL@d47Cf&^Vad*|?eISN~m*cNljVWLD zrS&$L1>aBce*c;<43daUH*ua*0R+jC56Vl3CG|Jn7S=ZVr|uE6;90jG93-Klui z>2qQW-_tuH{^5r-=gs#15Hs=R-MPu)k#_4!>L z`Crub>a;4Ggx-%;_Euaxar>cCMT6L%JwEUE$QG9RY%-MVmrQoIdM_BfYW;PMRi^8d zdJ6@ee0$tJG?(0ayesIwxN`6Bj^?`=EL&Xij^5CjE3omMa**Tw_=ahPE)p-Y8)GClUduz{%A-4{>X z=}f)Kb)hs(@y?;&LKCG@f_D=?{{=Np$!{+I zbiVoL$&Le;cGX$f1qT1QY|Q^%ed__!oJBT=c%A>bdZ`E>`Tt+H?rnSRg?rm}K7JIb z=lsoAx%Y}TQ$}|C>Cin&4LxO-x)^SG?0B(Mn<-h1eC9^PIWbMSSv^&ytM zP1m=VRV=V!S})lv5w^SOsJWEj)+u2ZeqNh?V%z=jOLcRf7uZJJ@o@S1;`PBtZl4dj zivBS?q<x#27 zdhZ20FZjl;woF#XrEYG+B^UW^21Ot1ZygfcnrfGmnDYDghE~B1|1-HSFZ@}^eYpD2 z_7Crs6cgf3R&FnADV17(xo7<&x8S9_9)y(_F4C9W?4P{;`Hp!^vs=6b?Zh4(|6RQE zNb}7-_vT9!-?&>J$1N@&^#9rN|;t&_f`+q%w`w@AA7 zcE&0Grun}(imx2!%dWh8ZRH~#?~1$ER^Db>y7==>=G49^7R=ne^Sa_)AAEIIn)s!} zO?mQ?BU1%Dr-$4)wMSp_`uwn`ZiYpnqRalphW>Ev{r=+qi;K3+CqHlt%n!cK?c-Kw z+IQP}e{cu0jM2^UVke!+FB<6>7TC0{}Qv^LmYKqr=Gsnx8g(d z8>Jo52DdUfuJ_!O5ihGhd7MY@W7u=+9m!Jrn!Z+_k6*5LF8+#*=)c?Mn=f`8@3q@! zSE@P5aQB-i?ceLQ~#yOnDLhF`;?>#B@3KnH%3N786}8_-?{!(UdhWSA@47J~!d# z#B{r-NuO$VHtE#qg06tHSWY->+J5*h z+u>GWpP!w4UpEJBeJ1&%Jf(HpEzZ>j8w~ZH{Qvy#N;1=wNjqaCde8i44-fdNc0MJj ze~o5aj=~WQh3g9p&kKhC+&Fue>LV@rcS_yqlX@TOZThvU(rsOIamCK2HCuPYIEvZt zc*pOf&JixKHna126=#8WZJ`unOkkS!d?S6Oj*K@2?Gl?lmOYHHS!^fk6)#yO`tX}_ zF1o~I~6!J z!N^(c?1@8(4ZK%g$FT<0H2pvD?YD zR7kVS)YvE=sm^Z?Uvp};%M`6et=ad0Z4vX~ zN%J1E{n{Y3;^wRKkHe}O!wj{WWm(=Xzv45a^~vqUEE^Ywb;Vy@VJ4;Dl`MGvq0KE< zA@LyFl?GE4xiVaXJ}y5cP+B2;N|wEtnUDW`r59IO0MD+l-~$g>KCO|Q_-wY@)m1AW zaBI#Jd&9ibM8xHtT7jHsJ@;jH8|Hw1hTkV|)u;Tss2I>E`R7Ue%6BjCY+bwJN0`c@ zu`1D+|6FIsbt7{J@5A3UA_*Si^THlCZru2YG47&N^GQCg9irE!8uW6w zRfp@GbTnKTcdA@-=7tA-R5imR;hNYf+WX8 z%U04e0?8x#QFIdBRrZdr!Zp{T4de`pRc}=7;7dP8;W}id>eJZ=5+v@vMiQ$tzCD zUFi~uC$A-X9JXWP=uJJZp)uG0uS(PXPh1LHel=ZQSSl>9z6W@Iljk_n4Q{v|o4NJ-=D8gDZlx_Ac~;>J-_&j%gC-W2 zttP>T+t9QY-9UL2ayu6NO^?o|s;}($>@%31Pgn4h2=vxlq z)%$zXYTv|uI-@$Z?Bj}mDJ7wS*?ZN+vE`lpf3gEhU(?|U}A ze!Jjd8o$mOn=_ZztUX_G>gn%`)4!BIoGZ9k@O@9f%Nw`<91wqa>0tkt)Q3O$7Yo*D z{g{5rus!B?Wy2@`w+m8?^4)D>zZ<{&O{HnDIYn3*hJcFiOw*>jLhn!2p z@0|-Sy)L$Q=l#S!-TJ;=?M%M7#I~VIi zLT_L1>|H2)$>GqqIJq*B*)E4Un2f@mFB(OVP||yx_04&LltVtYgmPRRNa)hllT60e> z`oH^3FGl za&${Y-x%+_urtJNtMjy7$!|TjM=1V%>;Lg#nf?*==_y;8XZNn|Ipw)R`x;MoSj~o; z^Gr76ZVt`myubcXnbgGZiWhdy=qr=kn98a|SK(QxM7^O!fm8o9My@BUVB-@hRB zzwFXV>VoSnNck@>4(^2iDw@(%NqRsogdxrn{QWQz}BG8pPV#bL@(m?Ia@x@nqohm z&p%A84=i7<_%Z*uzr2+M^MUE@h8z94#N%sQJDg|yW1sG1|7jQZdja>UeJhn+61Mki zZ}`3JVV*}!(%FR9y>3sGo#q`X`11AT*V+B-KaaEYHq7#Wa;)26zpv)nbNqWaS04}Q zTx9gN>%)%Bmd#SCFK>o)R!^IEUG}NjT&1;T)v_!`$GK$79$&a4^;5li(&F>?=RP@l zY)0~u^FKLP`o2qOzW)1AwCa38uD!iIv(r|%C7zYfvg~i~weIcUbTzYZJ8bR5mL@V` z(&ueAxB2O?-tRggaXi)iLyymHWw*4&59JO7g_T@zQ@&L7WV&;ncJl51s&ke%i#wL< zt&IwrQ8iCB^m|&xmBr5Qo=kR|bs}|IzUhs|%@eS+0Ucby3hi&s3u6Hy(n#BRK5 zS;g$<9?8e856qQk6MmQ@bADmD)WoaG=hM#|+teSVHt(ydQT>shDYs@F4mA3c(|+7m zNvGtA#?~K=9ZM(a7`8YU#of-gDfX5!og~p)67_B3@hb`Jo%U{W!O1(jw2eP`-hXqU z*!Yv@e8(qjo*wO41w6}DFH0VBm-t&Fm(G#s*tsNeyVTnS4^oTU4{#UDNa|Yh%2zq& z&MM$~b^p!dB~#wdyUERZaYvmD%hOMiEPFF~1WGP><*~^k1+JJ4|0YEJ%ae$FdrflbImJ!4dSj%k;}5xRVahib zdbe@aOHp^0F2gm?!h7ByJIB6O{PuR$(xw-RE9|=$hDOSk{;@mpYQ^L4j$se8eyOZp z-1Ut;@@Vyc|4m&p_CkqS98`QDnwdPEzdBgUwkEtcmWYZmafl%w? zsx>aDNAsj@m=9GHYZf_{+KTH6Dxd!BT5;(e*P=qX)pH(BOP>{zTzqlf%*>#*uJ22) zC7P=3eDK$9@x&L~yZ2cHNnI#&DS!QE%7gGNJN3>~h&+uoUC1fty0k<|weQpi4X&&! zN$p=w)mW|3&p27AcJ%)3pO1yN)CTd!H*8CNlKU$$r#fBp(#E%Q_jw-QX5Q0!9b zobbyZCb;g{qs_*f`}Ae($1E;M|I-iM#aepjxBDdTk(F$8TOiVT<@-Llc}$|`&a+>i zw*I(D4RhzYt^XZ0RxN*$6+UHgc<>tqqptNYkMD`Gtt)>ce0$2&b#K%U_G+zVFz@r+ zQn9?(D^FRz-LZL*ANpIKUbSn><&NJBA@ZNt&l^p+rh8>K@9g-*67^dX z4{p=`_{i_DZk7mJs>!A}9V2}uhl*&eiE|CtMju(Lf6s5m)d}CycEqbSv4mGg8}HG+ zGFj^5LAKK?)Q{f!S@LQ@)g`xoy}|8f=g!Mo9+TPMFB3cMy|TMcU+t5c{0E;;EG+#0 zqH=TR?`FfdGt?$rb-ley_DVz z4g6y$zIy$ez}Dv>+Dgp3_P>dajaHKWyCuIgx0Ah~@p3`Dw2zvbaCy@^h z-*|br|H}Ep5o#0MT~F8PS=R4#We>`FuBesWbS*Nw;=Q-XAD2sqCc9NlTCu)PZqX(0 zEnVS~*^LR_^KW17l3KQ9;e-FeZKBFcu^;Bjrt<68Zp_=eeP?Q7WLp1o@6Uy6Ok8X3snkq$ zyDT5CUmw5AMfhw}etlfs{C}?w1*9B{tKaib$=J^K@8{dcGk<=3`}yhH;}*65*t1O1 z=Vo2fbiKIl)#Ww2y}jb5)LS)crq62ief%rqP1`higDKH7Lz3ng6`J1-?AAFQ#}+f| z(^ZFs8LmsdYMuJ{?TJ$Jwz)YKUk*52UcKbW?dHuZUK=eE)^~fm=Ba^-_G=59Q?m`+ zFXkkO$+o7XJYLMsR>iRSiNdnLl4q?K`-1JcGS!*S&3UE~#Zz)I&?C&urYa$UZ-F;o z63CEui{d|8zS-!sck{IOE~XQFR2=7BY`Wf7{k^2}H)G?;FJE3x%Ilr0beOgA!|SK* z62(6YtA1_zY5jD&N-dXy>_0C<5u3PAL-qhjJMovuO{qh zITN!@!jnyTL#TTPZ*0_xLq+lmy5EbTk4$~L)S<<2U1M@~ESv-8eRd9eASviFgf5GXF{oA!yteBp*N-#F?$ZCUCqOliWIaZx>&i}b|p?SnkH@_acn$`!FeWv+Y zQw!f++_HNuuV3l)cb~csCe(yl3;AdlKbzoXt<}NHwr$6Ra)~otW%-Y89rEb7`DNwN z48Cir!JlF-tb3a}4cb$eOQuZI$|rK=VzMBU7G%#-q(wvjLMh=6JjR1&4{iG9#s13Lp6u5Dk|R?{DDab)fFgTX+WO0;hr%vD zyBXxub4BuR*m-|8=MAC4)$U6}nm+B=c_b%=wbO$w`7`q=(H*LvPI1eanlx`rUpPa; z^VP-9d3_aEr}zDM9hF^v?fYHkx`#EocF$_cdk+5$>$$qQXrFbOd!GecEZZO1&B}IX z7V0toi|Ie`JoUi(e=|-tFO+eZ@M*zuA&o5>%lwP~YJRx+qO#RC#mB zU2^%US4(HK%1x6m`>^g$*%1fjd!j+5wa0dIyZSj>n|P;L-S>QTOUuJAZ?W9vs-Ew& zqb}d5lGkDTm*Bg@&iuxqRkB}h$lf~Hb2K8U;k)$ebax&hGwG*_WxEwwWz1HYCozGplgzgzJl?X3NYA>h+CT9+Es=ety;- zoo_R#f$u0>8y( zBp!KFzhU>iY#23-11iN&cP83ldEqRhk`U8nq<6yFdoeb>C?{FKbRbOn9Sl+?VE z%#uo74=#O|)Z*mSyp+Ve5|Ex^g=j-F1p|ekST6mL{LsA2VhJpdYRfZJ-csZfXz<@>#Tzxw)Q&u~Dpov6(`&xsj2cp_xUjf{CF*w3(rS zo`tccg@QRui780b%uLVH(#S}`Kmo4W%t9gB)YMGR(A3CS0j4ch!Q4Q>+(ZE`VQz{r zGgiUeOu@oH0cyK~g`t9l30$j!g{eZcv4y3cp_zdZ!T@7qV?6@{a|^h-*jSK{V7nhd z!Hp3si1>iSk6UI=s*!@BB_wJ z9rMz2Qb8#nWUZ4^euYA`u9=a!o~4ncxq_~-xtX4cnTdshk%6(Xo|&bgiGrbliMgJI zsi7H)KS6GX`qL*hFTEtgP{G*D+|Z0m-z_tzB(+FE-z_JxB-JGqlzO@JA!-y1%}ouA zQCeMZ_g2_clZ3~GgCc{Jk@5NJZdvJMd*a1lLv>pim=BbiN=Ci3nucMY~gvL z#NM%+X_JP4%SQ(NDHV!MrzR-0c|WezuXwO%!6vQME7t6I#G3dl`tD~RK9}tZ7JT(?=-sQKSFheRc4lE>VBEmNav~$A$jnY=qm4|$3h4s@(eGCj?|b=4hBc#$ zv14w`^Bvm)PJEu->;d+i2Ed2b)RI+ygfupx3k!;D9(q@Q22Rrzz7!(a)U!>%nw ziQRwBA1>d=I-&jXYWJ{fmQv5(&8g6K_>tJKA-dtE-~FFuxeSrk))wF0eV4G^c)jp^ zak6CHuStE(>F2)8F&4>srm$YDv4E{+)$uB|gB8qkEFaHI(g=A~9e*hytoeMsqh3f8 ziwMsIzMl-MI@_YQh=yt^l}urZUg%vREOj)E!&M{bxYNupT4$8v4_&=sCZD)RfZOnq zumSVW=#6)Je(+hHs^yHTyuZ$W4rNOLG?#J&L(aCu5#(*jqfZu@tqZHk6w1|OY^Hg zm+~X#rcAi8)Z5K0uk)Q}bZuaJsI`TE@_7eK?rGPRWE-k71z!s`xinZjIk|LtgLI)( zpMBg+^ADAeZK|HP{3@QS$8%=c&%cbNNuRA6KbRdlShs5~e>S^=u?WZEo&RslVHP}g zVSA#?t)qvZsmgx;cZNavv%$G+{U(*Lg9f}6+ZWBNyyUk-`t(eWoiei;JERvnTJ+l8 z;Jm)l>_Bq_e~H8)|AfW&ou-^(v&`5Z^ya@8i~6^pO8xCx%bLGO9})X#;gT3H%JI42 z&Go-!N!t#7N_}xX=*ineZML_&m=3m@?T}vB`Lg}-a^u;H@7Jpt+x9EJHrFSD~(ZE_0=tP|)j?QymLc6*Vy zd_(od^D|f7Ht^fu==sRqz_Wr|&9htZe&6TYy`>iN9|fLm5Sg*&Wt-xQ(J+`m!O=SCf``_(f z|2Z%4FUbw){c`=1;(GDt#n#cnukt|&{P@$KGv4az?Ek!%^}6P(jo0k!r^!e5 zC0kB!il=JDzaxIBxe%vodA> z)VsJR9;-U@a*eG+#PQq9yX+4*6>?mP7P!x@sAZ}XviCsBp8U_d&e!Iy@teh3o_J?z z_v(j&ne0=H;x|@a$ayb)=KGFki%m{Hs54FAsb~A%m0ob}uEp_i%YFUFg(tuLGB5FS za6jW@l?Q($_?ae8e8(QS@i<#vcV6D&uQqKL#n$*={rzVTR{?fUXV*(IwT4EII;e*Nh2 zj&JXpYaSY2=)77{_d{yh<&p?ZpF=yWJFD9oiXy~<6E_qHb+yL-__sOek|W>q?gO=d zv^4}z2ketxb%bw)&>oN3g4dF57R|ZBx_OD0O?qteUkwNM1wCS3A=wlBzS|xA@k3&N zg|b9Px8mf}UzyjfmY;Es|Lf&S+gsn*df1ogU;6tsb=S97D_>M7{{3MSxnJXWW#5L$ zyTsq?#?9uhUUp&Y2Hu5p>sp?F$cu0?dytr_EHd#ihqP#gkly18_Cq2XLenRNa;S1F z+QA~{FsJQDc8fEIwBR?zKf!0(q&L2QE?B;)nvwaJ&NICuH~wiHs%!E8ll!-^w{+pJ ze+vJY*`_S@i8$4;?9ogE(Yn+B8B#j0C&`_2s8-_OUf9y8#UXV?Dr3R?MBCGz&!>Le zS3La&L-W76cK`WKFrIi|_N8&}f0@7k9ArMqAGlZ-ktT6HT#vgT>6zZD(a6F%0XuqSgvFm7rmVW&C1M%-yPonh=83uo zc7`&f&u4!zv$v8X|EjHj(p{b3v|)`I^#{^U*3cHYDCdYbaN zLq(iUg46CF{1m~`^(CW4m}5mczg%+FrNDoTEv;{CiW;k5{MY?A_xJJouG;yG4wm-C zi}pRgulO(d&G+Z~4?e$d`sd&O@*DNjfAEIAX-MB^F7aje?>`S^kN01kzhA3B>Y;Vv z*6%X&G_U@V=}nKUdzSXxo%QqQZ;EyER8FOL$!y+iY`uRIh)}xM@x$#U!>h|Y;Tx*$ zE^|A_2Pm}Zb6oH{DipLUNMNh-jr1*Vk^>|3 z6xialXM;}vEX=T-6ScP(CoTSo9rXFpk3w8$pp+=TYi zi=W0+u8C1FKjHn<{mJns>T4#Mo{>uI-K-YX9<@Ar{l<0M^Eww7uG;Zn$AujqcC4`Q zzB6IRlpUuZ$rPTekjn4ae(Qi+Zq@9AXC2DbD)zn+F35O3EnEHe$=gqO zqTY#axs%9PrMsO;x4iqq29F~Og(nr-XpDoY=;lSWZ`lDgA1`-61M`E|b=MCH9^IPJSRE;aUg_D+EeePb-w&l-f6(?UTIz8j*o;P+|?`1E^zwp;-*LRIt?r;|aw|X9`Q3tvfA6oi+V<$i`uh7tIfB#X zIYzW!D0}m3|Moho_m__z|5h`v>diF&?KQg|PUf9&^W~&egTI;aO@H6{IzL#9|ENpO zzhN)=&-2j5MT=YAi~IYmtRB7iuvpT)U8dJss`g676q%pTC+7Fnls@?}@%)S{6T|y# z%Aed+wm3C$dcRFo%I}GXlpps!k@vI^DnC+{5)e{;W#)?g(({f^zPI_+*;l`>N?-lH z@_p$2#rvZF?)v5VwfU9rj{3O)y;HAwDTHj=>g~q!XI|64!|cqbmVY?6WKWCMkLKwH zo)O=yb+;=#xToYNw7g5S-Y0dR z7;d)geBroA)k?_W%M-;<27g%WC%d0moMK$rlcU%_d2NbxWs{BPyhC%Is#$xmzif$r zAdsn?eZudP$xYXJim@saSSH?0;pbFi6LNBTDDh`*i=gsN?{54!nIVuxn6xdH%KZ*S$S1JE(f{M`$9>tR%SY_1PA29zAv1{pnz){0g-`sz) z{E5gX832_F`I*!ZFR;f?|)3(Xq!8vp&v|1Yw? zaQTb-FX6v4|2fq!O@9%2OZi0$gJb`0?t`ATN_JX#V)DYKyUA zkN7q+mp_~P=<*}IkCJ!N%et!z=HF9)Cq7?vzVPm&&yR{95kETp$n>M(N5YR@KXU!3 z`4RJ@+mCENy1$G4Nb@7>N4p=6(>;X@7q6dDx+p#8m-GK7wf+#;w||L5Z^rWb0Jo zFXer+~kuqyJ8!dqMY~nR8VBi->plKH^Q<9H4eaXr0hH zu~SDylUDEW*b^9|aZg20EWB&=k*i6wcew3|+!MAZI7a)P(!DP0Pq{Vz`~2^8q|aji zkW#$8qQ7GA4$mBM`6F|GB`QWsDE?8{kiR#S;h%mN=e!e*^QM<=2ozAA8nyT;>kH=Q zCSJvbiN`*0Otv)AvQjIX;JWSDQS}=_liBz^{XBi8XU#LzDXs|n)-h+|Vz>U2dsA}e z+1B6r(R65$xBl#c%8#PL>b`n2jY@BQku+YfqdU9w>eZ*$&#WsrFz1w3`qw*B*Z2I~ zTKbi@mwVQ%=qPc$U1c9j>wkUz>ODDexU)H7XDtmu(cmDs) z=S%JXAL`cM`{&f^^?Uyvo1MS+?{#MOhjYvCSO34jUT^*9p}Tuu-Jg@o#n1QQLjP~x zyqLHBdGB78@-r_2m5VoRZcFDo?IAyF+N?=7DFs1`bQSH+X+^UgJ?7%>I?Gq^;js-D z1e^;37T7r~zq%rHl12d2%GQsrqMNKY9@14i7GD-ArG4Cb!=a1o7ah_~4!!6gw>{?I z%_og>%ATyT>%aW&@s{sJLd|oycV@0EO5F3NZnr|*JEhIbrYP;*rm)YaZ2prkE`IAD zmnd8c)8;zS`fL`{O<%1iPURO5&DwUTi|I|CdqZB$pNi#rx}s$-m?rQzoXYLWzvJkW z%&ER&Wx*Fdu2~Ktr({Lr177s?=qE_+V2xh5_*AUX(k&YlZe{6udzViBIQ3)k9Rd4u zvU3haCfLXyTPyWG>~ms@_-x)a6U&#BZNGabwZT$U^Vg~$TF0+0%{v%)>==jV!9wmk z%U>BJ7o#`J)L&vpHu^&96K6!znD-(7!DE@$sYAx13$=aNT9=&cs(q+3q1z zYPWxva1lL~d)fQ(+IM?i{4jENJj18F#W(hjWae~b_H9d+zB6d~!LwUgBSP^@<6qA7 zeCg#D){DEAe~X%C(ECU|!C}W!FG*9)-s6_NDg4hpGFHoU=KRo7)_A(ksei|vTOUPp z+b*t|cXGF?w4C7Wx+7QO1nWQDZZzdv>XvH!hodK@@RD@ajd#oMYHGfFk|L(`>f?*x zD}_4x>C2buy;-FENSfD3$wYq2f?V0u#g{Ms3DYkAFiX$+%)zGWn^S_~Bik%5$|m^x zYZhPTx_0SpKIi;{QPX}#R~^-UWH_n(lXv`%=*c&J$8S(nTC*W-%WRiFj+-xEkF(L; zS8-}h(9R8Ooi^rv{KBXj|NF!}v(}>aUVYI;E1fQ8G;U7{h`W3G4A;`C?HPyvhuExG zeTRunb-hB-#M=RdGv7xp+ZA)esTX+Q?algvbUbz-K=&i zwxn3nZ&6#1U-REXdj)(rBKF)pbayUW^0K|&JC++;=v^IN9B<>;yRO^;xC{ZC)^pkmBBe_6>$ zyG551&gq;q=eo`$ILFEMNHeQ`&Q}ADLe)nRM~Xi%$e#GRrJPOLX!^s7$`kX{TMi54 zoI5_bOzjbqBG)3z=xp~1`E#!I9XvVnr&Z#fgl9SjofJ|OuW`KU^Xc9bzG3mQvc%Bk zW(_a5uR1I+Z>Kyzf0lvLb)zRnkxk#ZtvuWljA!0k$$KVd`ugL(8)X}n!fneocG|yn zs%0;0c`V9hDA1Vuxix~rJw}D+7033H_oD9}&TLGsi}hTyG=B^3z0ejSy=x_J?B%v~na;Ym99o~hxq4%h zROO}C*9!t8SY8X5De5Loe<+)A@a`NBi8(u?J4DW8a2}4^B75-G3cp-|%V+KgN*%j- zX6~;AXZHMcGWKy>u%gC$`N=sqH-vjU3-)|#;%?k{jBl-+Z~CX7OsvW?_`h)*9#Y8b zjL2gw-^Y5IrHPeWU{8fy%C&{>FFK#u|Ak?iWL@yVeC;3W?cTgy#%^6GQY~IP{bBM? z!!H(PX0z*fW=~jDqF*a$6!1X#*3m5sIKZZ=kdMHo~C!}Y9~kO z{`JZRO}^9i-IzFOoq*VErx}Yxl}wb3@4Lk>IPc6;_R;(ZXUF{IcLmyiSRP3`a(4^s zdQGukCTH6WJIb|wsP5`NYEs+wu4!V2wRO`Bu6KN<&bO^Ycxwf*_SRS=fBqUdwNki(TM4&-82Oq*NCrBQh^|1r;D=uUeQL+kOeNRULa!n3WrXvDIh}c z;Fq-yom_f>O+u;~AR=ZFi>g-uh}hx6={0463+MC=i(6z>-)JO07b=?qVaFWm1F?%c zd8V9j01*#_BsC^Bf{25vMgdC9szy6ndW4L;PG~3=b4seFOqmeq^bE#l_L2xV)F8yB zm*~I=W*<=HxnsYU!{?ENmyB~~vPNN#H-zR~n1|I}Z6!coMsLy3JS zln#8NAP(XOK3QP&RVl0oNxJB|O@*DpwWtYk9}-_^tO zutKxu^hY6!GatWhXnt&eIIZtL%W2Dfd_@*qZ5eNNXDZyh z^Rq}H$lkL8^ zw5RGX(v_X2clVGRP866Vdy^LvRu1D>fG#yD`lt6JYlx@3e$Vq3pE4o@i#zGap@8Gh=Ky8xt1_i~h2-cePuN_eeg{MPF9zAfw%Ebm{MxIX#M4*NF&8)iRTt~)z- zweI)C_#@j+7xkyEcsgy19&gxs*S+DN6|Hnme~jC3RxT|v?c|0nOKKA{x5eZYMl6du zl^bzCap$c~{VO?BmrTjKovL||PyK;Tj_fq=+}N0-N~e>$?@mgQRDGZPt##tQy5dXU z47nYa<$np;lD}zf_t76cbM%Cg{KMu4tq+c0Dz{v2abNiHmFz2%ue$lJIJSWE(hF0o zUbn@UI;2)H_HH~~eP-UFeI`b$cz9O2d=s(EUGV1QzW76@SyiRDrZ-#rXe>K=`Pjsy z@^V#?^Aqpd`*y!qva%`j{`k1VdW%HY^8Vs}AI*0`)ysDk2X)P(~HmIqvK*t1=f5s(gHFlaYbD&Kcw9t$nzOisv0RdvUy3uz3@!wUDr7_#PiGNpa2I={u90E8NAnH!X>@;67L^`~HN} zkA!z$?8IgtnsoQ{q_f9`JmmY29#58e^W2cnZQhjsPfuIwwU+Q@$9_`!?XXa3--XVA zhjE)X9D5UI=02@lCo|M8(__ty;K;)p4!_LnEf?D{{fPJp>$bfWjnBj*KX3TaB>N%h z;g?IY`nTq?${v1mnCr1DYwFy}olAc*AB(zc9hUiy)#`NW+^L&3e)^<;Vb1wkpKnC` z%vn=o7B*9E>!2ekjNA65`FrZtAy0?O%3;Kk7dOP?R96_ z#LD!+YJI=_b3T^@-^`zxKi6Ho;ud13X|ty^c>Mg`Ae z?7DmMa@Squ5C8T$uRXReycPTX1S;|wV~O8R-=ZI6eirgx^=ow)s`W*?>ZUf*!mYyH_hzs2@D*L*$uAo-i=vMs+_ zqeKPo-8khsVahiLp7!GkZazo2MJo1xXg>b*Z_`#opZ&*7WM4Bsy~^l+;QomOC1-U@ zX0;!!E|U@s7@gmz+f2`GFWw-h|N3$wGv`t6NR~4VoGvc&M0U))^E~#2E#ve9Z$C`< zF1hK>ZqeKMt>F*EuRIXG!&aOi^yxf<`W%5i=b7K*E*$XUIKX{7M2L~^2g7~)MSSH4 z=6=xmx%s=e%vFxl8~4YYXs>zy`JS9d+w7yImlEGzE|7kss+GU=dh7A|78#2*@A2=M z@1!*4;NcG}uckQdI(qs5*XM_BJ7&9ZtykD8C&$Qt@p4h(o-ZAb%<1>x>4}AV$Rm1;(k@sHjhek8n zl=u52b@~Whd?bItV^@ow;JxOB7PpJ8Y!fwWl(yZ({_RlRma~EuY{xq%x=cQOQl?1y zr})*&T2&4^{BLOP(HHtyxTW0Y#sxRs4;pVC-M=KOAoOj%qt^+WOAhapzE4iR;=6yT zeYey{{YQ}>=S+EF`ONb5+$Vnp^{)gLu*R@^U);w3`Mu^puJ#A%k!f-g>x+8Yoh;n?DC9ST(-j2cD8MK#k?LXdntsyyN%kDo z59Knl&fEo8*lx9yMfts7Q2yuz`;FrgUzC3t{%!lG;o&sLz$rq(Mq@{--ophS_%AA` zbvYkRe3Vi+?SES@QaV5dQ&ZiSO1I+4C2;{_m2KZ@&DZRU=8&bH#U!AeXL_E=8(4XIY#q+BE5o z-#fMM6WdP&TQJXI4qg0C(`3HiV!l^Pv|o7bn)dFwm+&7=3Myy)I zH1F4uKpw*`>xW-55_4-ucv* zyeFQ!eA3S)lG`RuPBGaz@5zLvAMAPsmQ56$@Vd66%f(x5LU4oB0p29hn9s-WubdoM zAmUdcC%}BvKDq8y6YqJNfHSgJT{o#`Ryyx%t+Js)&fFk>Y_dki5mRf4&p#|u)z02H zU9{z%Tg>_2g}HMM*%X@EaJO@+s?7;h?OT7auIu8t%@eDQt3K#25zby^kga~$+~U20 zkx>5al?lh^nI&9%QaqGghp0zR=3IF~jxvoV^CXr>zhWgegcUxy>35jsz zFWf8K!FX(<#V3wap%1utn<6AsYyC8qRH;?E2Tk7~|6M}=+V;oqM0uxAU|r#AaPF~8 z%jA<2-p=f@`VqJFKl_HS|M^yYWNLWr_v7XL<{w@BAt(PAUw9xIFU_@J_7mp6YwS2u zE?j1JG_~WH$CV)QympdMAhYL#3FO?gBx*L#{h8IDhy8zOByX zysyw8b?ohuBa4H|FWOW~cr0>fm>cy)pySF46@Jlg6J0I{nHx4+9{mvfZs#4(w#z$( z&p&mNyVNFE!*hB=TZD7Y4+RCMew~uYz*d03+IA|uHl z!l8}lNuS7bJ(J}L)1L~Tcf5CG@?oR4)qsPenjT=mN1SzbUj|Lv`o zi#LTde_WUrzaxu1J$>by9|tZaR(G0m-u__7GU;ea<_5i3d!eEwHh*nqxX*Rk8nSri z>fc@*`(j*IxOub84bxt=wdKSDGo}@xQGrubrv^?6k(z6(Rko&&S5QE7_R`IPjscx- zmV16vm^#aAk%z-#^O=hlbX&b)6g#wECEAVUlB@8WsP1Vg8PhXYmht;C2wHC0;)Tdwd*UDt2 zdE}lr8~gK;_hBQCjn)ZK8>=r&iCntd@k5oB$)dde>&?qv-drf;zA0QXGT`0svg=u* zAAHv&HYwOXuPnQ{Gq&4y{RP96kWbR}RvJx~F-zeVRYv0F&PoS3bW@HH=A)hbjx#w?|vvQJrIB-WE3Zw#M6^M?b&N zf1a0YXmb4b(k<~-{LRlc?9F(x;E|*8Rc=%6ujW}{FY-;+c3fNKv+6}aaK@Jh6AnH* z7}(67smHy;YtFj4q6YqdCRfbeshSmjLGSSD#`O~a(tcbLP&u+9C82MtOYf$H_ZF%` zf|HkBYP;3!wAra!y?vrV^$ttzHQhYh)ZLe`ayQ)Tozr{ObD{A{*J-SCielVkdb@qH zu1`#u>pFd*a7pZ!XMV3I`R&X0whCf3>}=|>78O*}h0KNhYkaDMUOjG*OBC%M?5cQ?O$vA2ny z|Ki@>Z?Qs7ZOg0dHswTrJM?k=x2f-C{TJ8UT{q4C`Z?uY^>NRuZ#a+NKe2M}zvcCn z=c>;6ow(H;msp?tN&HKJfy-f~-M*@C&EA=DPxne$_odOr**T}t(NQ2bL5`2j*X2@~ zX`=UCTFvm7yO$P84}=rev*csA*VdYw6UqOz)=E^L3t|zKRWg4561}yb zb?fo>TvltYt~Ak_>nXfYS};T;L}bl1i7kr96p}YwG*HTL`LOihTE}kQ_b-{&U5%68 zX;3z`EcA)*My=eIz}5VGv&61QZBcr0Zb5tV5(UeEd>)=F?5__q`wDI|Xr34%al7|Q z2FuM;XIk61*?k4P`8)+OIF>ZNcURxK`p&AntBzAwMBS9|fAXWJLMos4eg4)RiBoIC zjHl@TSo!D4%TJU4g3uQ_btl_qaH@HQMKhO)+jxE2>_ltmKmzmZ;I& z&(HaCbJicWTOZCYOx3rEFZ{d5?>w`-iSPbpR^Lx7tlT+S-}}bTrCZWJNuQSg8-3=) z>}{2H%9XcIt$A+7HCf2(cx>I&xTK%uP2aT(S6GNWQk?9w>H3PrCWb*dE?1sjoPN}2 zgREhV)FN5INRi6aq|zsMj?5Oy@86@GRea^$i}1y9-Pcx!G(Hqub)rM%>VyxQPPl$i z+ZFkB%PY~T|8}n7=QS}?EROKVN@%i8%?N9<%`>o5O5Q9bR3((*6sQwDSJLrd)%!XUZPM_9%%V4%U6Bv0-@7AX+E3UD>3W~gvBz^qG z8N;F_O6_;^cqB}80*zg)EjFB4bbrZmG24%?I`7GeOkCPOYmVzCEd%vClN>oD?T%R8 zcCj;^cyigzWo(lZx@6v+3ZGVf`N-tu;&XJ;vu%!s|7yzZQa5~=bR+qnpJnX4Ppsb) zPI*r2y_oJKpk^Jz`NDjg#3Z*KiE}mT63RYdUqUB&iONr(G1oN7g>T(=oozojqtzZB z>v%Xb!c=Lt#a_+#ayPDr>bTstiD#F%awxTbiv6bgXSLh+*SLB*e=I3F+p%7_$cuZO zhD;vEg896Azn|N5^?b#yynP?LX3X*M4GcWKDl)q}Ij88%ltYH|%AJkRdhSe4OH+HM zVli`l_@edipZrv0dv~9A;l21bhnd`-y2MNQI;XEndzI#P#$@`@KWoBLyOWuYx1_p@ zcO7j!n%H`=eAWAluf904@QVe$*&aCQ)(IV1F#!P{tI46lS1metjCdA0B^msFyJ4Qm zAtM=mBeV1AXIXc=oqPK4ag{UD`_>c+-PcLhIdH~e-xaxlwX3d$x`)h;H4LhhoU(j_ zGt(_SlOqfn8<%`ASj_2bawvWCRF$I28shIoa=*cV|oP_2%!JWz*Qs+44WO5&D-mKS(M@EJ`d;_zhR7YfOHe zOr`i<@Bdv7ML*x?k)Pqe!Y{+GWiH2mS5Y&goRl9=7fyE?shJB&1kOpxr(}hpc1DH5>Q0#{M+*x)!XC&eJvzDKsGO7# zLs<%|^{0a$N+nvWIp(e2;Ays!bG2%i2Y-sog3=-Z&aSSo=-!EeOYMR;#XevDsZ%}J zZ@y3BZQgfgnmN}!9p88#%m_|RPE0gaa!(3m&+zPVEu8Uu_JUI<*S(&8PJbnz`mrgO z^XGr4aaQu%S1_k;wq@?G-01K5dsZp_y(#XYB-lCYhFo&t114(CbpK7F>Ax5 zF9q`hFSDPOyJC=WS)njsLufKT&$&x3$F#ETk8a<*c%pyuhIuLW8YZ*eHuqMh{a%nz zlW@pz^O+;(;!`W1-_ZYEpj%q55gnJkWL_(B~zeu=6{= zwRt-F&!Rv7IL~-`UxMb+IsEb|?%J#y9h+}_a4a(s`@c6{(%#GLVdkV3p5sS?_c4U? zC+ba~EVOy%dB+WI8!R}I4Ht&Zd?A@6SD3W>`E2RfBR}qW3azqA7oTQ*?5vm2y1A95 zd3uV$m*3ZZczN4SO={Ak%B!3I|1zCE@z(2Uee(CuT)X(T;+V!^86ig5Z&MPI6c?W; zu)1T=*Tem@<&W2vEKMh`1xpo86@?T{&PWJJsYz&Zrpa}PUWuA}PGVg*1IGo`tSx1h zirKrTRcv*UPEGsQ-Qj&ADm~BqvfF1(RU!T_Y z)jai3c;);lLiSTP@P2!;B2ZCn-egl9^<*zKos~R`9?po}^g}4na8ge6*83X^i`G2% zt~{mc`t`8wj_i5X`(AE37P@`j?q0s?oUpQkuV$aC?VfyW!K3Z6DmTw&UCg_%D6LX) zm(A{NF5c5K@7~IL>-e@j>g1xe7xb=fOKw$Jai!v|P*dZLhYB*zCyUN3*795`RvOa# zy61k;8qUb1C2ljr@7sKs(xA>#*I}k%?H%kT5qzt>_fw6h($QF#FJI>Djq$PhJ@LQY zG1EEQq*dNjb6obvQ*CbdikS zxzDruoXEq7?He-=oDpE0{U)t+hGx#RWp2u4ohsgwQ#CG#u*^5`(0cbvpfq5Cm%M;O zfyN{W#yejf_S`u6erKHcgcZk?uTS8!nfcYNVcwiWdutL{B>bfG1X%v`Of3(xbdO}; zaYyPMH_OH|pHmM_2^5YBx-j9=tna7Z&&^rbJg=ZnC-UOMjUAoR<$dcTKdL|bzH$ch z`=c}29`O}T|GR1ZiRrWEAFksQk6#-9@S;^v*Of~~htF)7DVX|f_lvu$cAJ*J+`Xm! zh2YB--zNO?e(8NRys_=LN&cnnA^KTy7roccy1GhKO0nhbvRw?q#uEcWq}^98)>+vN z6zV+iOy1$8(WO53)h8zGxvRqVhke<-1^-ss6g?78cy{xF%CR3SeBzCw8II;%Hu-k? zZ{$+zl9|eeENg5w8M66g$-PosGWY1$A3s?|Q^Vt`PR>~QGU-w5XYNh)GPCwy{cL;A zka_W~hGL&(U%#=3D6XpTouqk0q`v&c-DA5aT~p7#dQ3@~_pbB~>ldM~KmLeks_zQs z<(wnWg%;Nin}VW(iKQ`f ztn?H$4c+JGp6xr`cit~xbxBaiwJbiLOUI8MJ-S->(VLZLe`WV}9~S4!^ox@a!fQ#=#Rm->)@8DWcT$em z`If!8l5+msNu!O{6HewyH}StQzmf9rI`2gjDR5U4MJ4;liE=2Y)#5*X-6Vm~wB0{LvG9e>J|Gy>89q z*7#|?)G34J)|*N{=BitBuXmMDsL)L64*w(@bm-#_{Y~~_hu?Wis%UN5w@L4K8qfR( zm5)}b%FF(L@%P}o`Oaz6%~DPnc0T#J>C)x#v`~xXGj9{=pIhmFdtdZ5 z9%<`yLW1@hHp_l}dNez){yOiQe(f69{K@Z*>-^hR+reYhd7YP2PPU{nAaaA1k<1w< z7mFEAhkK5Qc=aC9$+&8@_1+m7S)OHOlW(hpD7-qhJ4sy7`UA(=z8M1N?Ba7bvcF{B z@^V*6Ly3`exJl6s_Svm2!7q9~*gRHyT(&qMF(xf0-6q2(b;^MZgHy+N7I(D&&c6M3 z`?;iYzI87R{~WeoyuCUl@}0{!$u}=Q#TXA9A8O_O7#5&{TU_^m&ebeBs>{3~zbOCukUcoRR+P)07!6`X{yf zYsze!Cu8yLp7Ok_8S}T!5M0*Txnptumf7ogW=d~z6xy=NBUJA!ujaC&it$0F_NQAr z7zGz_n@g2Oh0ZWEw2_H0;_iCzyNcyw(u02o_pQ77CGUk)k(aZ{{`w`Kb2*;Beg5=X zPTlllo>8@zJEO(S34Q^~oL(=qdg0f~7xFe_ZrBTj9ab-V z-p_v-BV{A}=l^+k%^&Wq%M{vv{h!~@w4|*sAY)H$+cV8AEWa+C7L98CBGVdnc+Q2Y z)@u$C5iL&H3k{Sa7nWWTtnpaLoDkvC?DRUZ!ezG;JJ)6>eytJ#p5BW7zolh?4tJY; z8o6|xrgRp}`YW$dx=f{2O!1J&*-X)Et9_Gn2^%sWZb7D_j0I zX-)exy~t|T(T1Io3k})M6kE!k{yKBb1t0nO>viYUZ*?;M_u6vqfusX_JVDas1y(a} z9{jG0_hPiB7(TqO1}WXjBLt1Sg9SKf#>b`sV( zk}i6Eq1^2`0oPZ13ioL%Pj{9)dHlT-_eqU|Pa^xjPB`VBsLN*;wDgkrZ}SSflUDbX zXFKv{#UHOON%ZD^zV2bN;p*)g$|j#B!-{T--HSV4ojBv=l>?;(n#Gw{wZjF`0UCf*NWci$q4_OdijpE__eU( zzhA#z%z1p~`i$2BZ{klTM}&fvgx_U?3dzOpUzVDvMR&(y?w$`q# zwYFc@rm}yZs&AE&ci1cSpWeHzQRgn7xIR-~>h;bg`D%X`-@YmxX|yeVb;Yv2?88Zy zpKnh~RlWZ0Rp4>&+Kab~;`U|Q<=0I;B~e_#zxK|hQ0An|oZK7k(;v#uKB@ocEni-zS}M=Bwwkbxzi8~M}zs_hf1(a!MLA9ty$dS=JQyW#&u`oF5B zzPu zA34v&)(2Keb4{L*Z}m>WmE&=tThF|ntv>rW=7)dy<+NsIoBuXGy`ys%3)~m3d|Sf7 zBHRAbrf9al!1bd1z5dDbze|b!|F*XM{;lKN-1{JH|opF1N@$`@DapS$Ea5pZ<)64AAR;-`rK+WD>AIiY*{o1S=l{{MT=Hwkrr{rhP}eayCB zt9LAo{adzXS>5t=SK{6+t1G-+?)M|v+wkqHivRz1WJz4Ouk`N!&pWgKZtk=f|6XXO zuwMWCmZI*>|FWx|Kfb)WzT~>*&D-hq_g7_wJX)-On!jhE{j9+F8lMNn|JOeM^W0|J ze#@zs7v!7m+nlQUJ?n3!U53^9d$qQkf0}-{>_2r?F=o>uE%j$&ceMZ7 zZirQr&%WI7c9OCpcTqw5l#iA?-kBB8PdJ@mC_6uo@hDe1+cv(#({x(TH9ma9uddY7 zkh5pLLBihaABsJd`=w8?0(rf6jGp;ltakv0mF!t$d$K z-rC;o7j!O)_mt+mwf^`0ZXJn^xx;+A#v_{Z%_=ev*J+qYaluO!S`YwoAVpT0;f zx7oOG-$K^aPd@Vc_VW7niG@4wy%P~_8KPIP#o`}tu+?jydDjANpURr`l=G{GdCrv; zJtj3Nd{aMXXFpxJZl|g4*-s(qn>S}J+?-Y8T-&|5uk_NEh`h1|0@rr0da9MaIXka* z)tviJimsfRdg5t8VsfnWZsY#KSE|li(R++9o=NPd}i;XoK?kFi~E+f z{k*Z3`Rn?eqm8e2riJPqzr%KI--=_c#%l_%nJsdada`p}tKYc;d*4jRz9>^PS2#EP z{VJC-_vFRGXCvZY9&xOBR%V!d@?3VIQR%r`M~%*{m7E)M`qzr$t3{u;X%t>w(-Yn) zR6M6Bd-LW~>AugK!j|P^t8U&ObGIS*m|S6yuI{>DJ9A$7`hBVPF4igzHlEdUTu|(_ zOtI|sov$`#r+cTaI-j8t_cSYSO{q?rv%$-%=Zl`Xxht<~(+ORX)V+FU>6shSv!z~1 zRpdMW-Dq{od}|@^*3iZ2iv+lDo>{1D{Pdu3>{MUgGmg#{-crdkE#_X1e)7`y#a}D; z%Qa@t?YnB{oQ>geR(F%>*Gidd=&M(_OwmA_gR zaX$XM+<%6R=e!t|c*De+hf6YBi~cPQ=H^xx^P45Ng8Pc^ddKu1xo2e#TOE7(a8L9O z`Kqr6w_Z2ce)3|(<{ABm56zFvI+s*g=&U_?O^RXHt*zI>KmItp;=qJQ6MF+69;$FF zJAOxP`Q)ST19tKp&61I@;Vr)L$ob?n##089b(0)lxRhJZte-V!s;-3W)r`M(duKeB zer9FK&EFGeS)y_5&zx&wZ=|l~94Nh!@8o&5ZmGeaYTt|6XOcqVUaZJ1J#%nJiAi$I z*XeRv=T|(RxbKEN_ngU(WMV}4cJ90w`EpfR=)baOMo}B*9oLv#8WOs5M!#X%tcq7& zsaJjPY_n0Hvpi(&>T^fcTAB;?x$^pco?W}}?BW^WW;%yvWX=tKowY9UaYDRZ@%6-t z#s6NcI(BrE{u5DdUYU;D$xfyBpG2ICzVIsN+2Mq3Yf8+nY8z(+eR+1~tSRr>t)UXD zxns|{?o;_b`=znnmnFiV-9qD&-po~Ab@Xy!s`vcH*NJD?FA{^OO<7g zUo@I!cxJ{l{pBZ{-P4}(Oe}8e^*Mdw)QJmO=_1Lg$2KHJM*cV>IZyd{&k4Dywpsz< zmoHtiu(GYaxX9J{^`E}=IV)c8SDRDhI5}BxGpn9s2v_y2TL$dmFI3J%@y>V@d{Zg6 zD0G+1ClmH~>9af1d(ZrtyYF&#)T^w^ucKbcmY(~v>)B1dY3gm8PW=x^n7COid+z_Z zmZ_ISt&83<%1`%sByVf-QF}(RgE#kv_-{yzDY!iR$qZxiR8i(|AubTUGXYh5pw_@Vt$)yFq!yqjiu;Kr@Q zq|l?4-vl=of8dUP`?aLSdfMJf-g`HM<7-W0pS*i>hrcI*`Fvz-F|YHRFPC;-uVb0F zFz4>88Rm(R?q}nVK7Z68exhkv;N-TOU%6`X{SU0N**7b^{YMsC*#Fb{o~1HZ%HIcX z71(HXzWcdVV4%`erp(QblPo8yXiYdJdnrsvwnOU6o|5U|>u+ZqWuE!=WUBYAPhK~g z)|__dG84Iz7|`{aMPkWB)sw1ci}m${C!|L?SY658{pw9^j82@e`K+=`ojZK*)TW2L z{LOmNVb%f6P_HOTP2)09FfrjWfCB|HQ&VG8g){|-n1!*0iG>1KRv`~2W@Kt&hAL)g zYHVO&jv;1ffTqsC!r0Qp1XZ1Zg^{@hx;jfsLqj8Ub;bth<{28Am>3#ks53G&MHMqN zvM?|)L=`izFg7tlv)9nr(!k6BLmlYk9h5LIG&M9aN7rj=Y=rJNLvuqDb2R@Nni`v# zqS;|+YHVSF?q34~19X3Y9ASoLo`Hd-nFU(d8yHxcW144aX@q94fuVtcDY|)v28Nax z<{6lpS)zrDp@Fe6de|Eonqin{WMqJ0uZe|)CAxo2ERBuOVf>Z^4=fsl4ocwey{eZ;u)M5og z1p|d3E;~D})VvgsIzuyb4Ip(O)kUd!B~a!1L8-<0rA5i9#R@hy`T<4x$-${5#R}2- z0WNL|`XQ+mB?|hUxryniPEgtzN_)m~>AO~xxCfUcmZXC8x*I4Mn!}u+@0Ooe0upsI zP%yLri|V@>Di~TqnMMjm24JS09mvJ{zNsmhiB9#pN`FULW!3z4JL7rS7LO~;=q@=(~U%$M(TrWL8KRqW^FF8L~ zzqBaFzR(sVr)un|YUHMB-EaoS~PRQmSg?R*;fbtZL-u>**Zi zZ(!MbiS5jqbs8(8>ZtLh4nC0x}9_mwGkd%~D9O73L zR32qi?35T$W*HygoD>?U2?`@SyI3xL=lr~q)Vz{n1tav>1`B|q-Nr`WEie@cm3r) z|K)4@|KA?h*FR9#Rr`1F{{Pp<>udh<*Zp`kHSvDE;*B4z_W%DqU;lOb{{LT(%m0(v z_wQHzo+BM^wcBoOI2M1;er0CMXLai{Ie!-4G@H@Ddo4brIlk`y$M*Y&!n@-4|9|Sf zXXEn!de1(5{e8dgPcg^N|EI6N|Nr-oQQOBZ<8!=AL>PshYcG{5pP=PXRB<-s7~iMa zEHBqfxZiSYW6*pYT74&+GqT~zC#m8oyi3#=FWu5Mdvti)MG*yC;a!`nrF%bkZGFnC zV#;WAIs3@Fhm)KtTQBlW&~Qkqy((5#vid=w*H2To*L*L7)~%0m`evrpdqMY|+tz(a zoV)j2DSZ)NZ{DIU+rE2+$AyQYQi(EgGyl2&m(p4Bs(t%~rp?E98L}GOo^;Ek^Z3eK z1+Og6ds>Bc>=T2cUu933S077NT=oiH8ONur)`(G)U2%jtR@1vt!~D|?$0a(hR)M`Y7shn8ygPID zw#L@*^H=Y3>{5SHyVyNry2Qo6jMax6yvysvc9qO|A(-aw@_Y7$`YB3LdZt&_9zHt9 z!{zx6$0ZA%f6J0p`qiMJsrK^K$~Ov=7Ei4Z^uF!Tb!UZ;e%9>YT0s{)H=2pfO&8US z&6nKj|7F_6bWzV;f=}I|++MoqES|K>{Bq}~8=b9J9#`7D_#bRD|NiQwne%!p;?(pm zXTFlT^i=t4vS?|r(~f6%945sqomAlZLRES?*Wd78Y>QmBTy#rr*Vyk>=d|;!K$P}H zGg0yL&1*ugF)Ms|%`2?=>`39pu1WhiCG8yRCU$Wf-Dz_c4U5TqCM&-5w!*XLT0z@Y zmFnkRbQJH>o}&=8eX97vJ@Y;4rmno;mdev!veIGU^OYC(+Kb)sov`cTC3C4wW@j^h zXKS)<*)+p%o6NeMyi5M97TNET#=CMquS*?QN%5zrzZx_Y-xbHMlkWIa?Up?=L{nX@2{k*Ad>+ImW4oe)4yLFUr+G8%@nf}X3`)@MyPZ!01|70c@ zdWbk4$=Q^&O*LJ*P3B83SLnzjNR8 zhq>QncGpbr>g0S z{XWZ6A?xE)YCjm=kEyMUev-~})Wo;#($)PESy5Vs=erkP4q@GKeP;BwzhU}^Pcddb z4SKU}=grGYZ{#i(b(7OByUi!F;bwp6rb{sllE)V=y=e8aOyWs+p28<>zAuexpS0gP zZ=QJlRmqCkH&SIzTAD1`G5x|^=j9L8N`-{K$JUoX`F-F5K55 z_C)H_;+!qs@)sA@Ww}g|;Jb86xZKLm*E;j+TVnw|o};%jCWLf_F0qBi5wk7Xy%U-a7VE4CLVlp`|-(d zuFe2c)EIWG^&UPxUO_e$O>znrPN?G4u`(1+fD_AjK zvs17-Y9N-~z1G@0&g)p@(ql_gwuLq=Gfr3R5@z#UbF5qT(Squk%&O;0GuPbM+w8qBUS{tiN%VZIOvg zZTyFJa{<+>g_WD6JGeGn4ewfVZ%(Mk@+-6W**xd>mUvk>UeQ_-@=5Wgm602}*J7(F z&D>V!T%S~)VD_y{ao-?(Q>3)znTM0odbK$Uw+`z(NqqKLdGbHoyQO=tJvvdRv|jDZ zJ=>DwD%;=x;Jdm|-Sf1?K4;IJK8p-azvav^TwyY%+XCn9^fer~$wR=6k1icf{b ze7W1J3bp5FBvX|Pcl}OL@}7Tgx`!J7%N;ZJx_acEF>F)j+Ih*2$tB5K?v_HyUgODu z!qw;Qxjk8A!l-2F5|uc`KxXZPz&Fko)+L%MTu%zr(-#W`EDN^XsB%+1_2tT)LY%jw z_)|5{{GKEfnVD&&6n~e0_bGczEz3VBx^|O=ki%FmV&RSTx zDImFesgbN$=q`mI&5O}3dL3O-CehKCct6j|tS}dhJ-1%T;fdRl|JD6`x!Fr=KNSj} z)L$Yr|8Zse3FXOQ3f;C=pF&QCX{hdYXPfQ*Rc|$y*?fg8pzg<&{aIu z^hSwir)c=|1;g z%6qriHM>o03!IlhUrRD~Pkb zVa=)+HK}g|mN?JX%AUODz)gQv)tWT3?~8YH-2KBV%m1x3%p>?&VSFQ36a%J8QJoAcRz%9FU(du@@i9uh`Y?q)QdVLzr)_2THW|( z`Ze2y&CUX!erkHXwm1~{>YZCB&+Q5c?o%7i7EJN0e$+42;-|vHr5HH%%mKsm{0>q# zmskds@p64ij#r(-)+*pL>14Qm?(>OX${z}!iLmjz?l(v7;$_*b@$aUcd|}n0^LdYm z>R_-W`>ZGUklYllV2=?}Jh&Tl-ppMlG9E(Ms*9<@plw)J(z!{DCf&U zW2MO!@1!-w*}}~BJxu&)b|^=eBlX6P2?DCy`rB34o~XTEEc*8FmaB1_8dsz^N8VMv z?Y2sd?^(_1dz}eWuU)i>dA8#c%QLMPSrY9!HLr|f9#&6w{osE>MzYUkV|`HDc8*xx z$-dE(|EzfG`&?<2pSW4(2Nv^1}>m=csV~r+hu{3u&MC=sHS^|4{$@BWmff^9Giz9X%foKRI(k<;Lm>Q5{TA#5tQd z>UGX_7P)CXoRFK^HQC%cy`)fc%SFybYrb}cIIZjRIQX|?Z`Gx_){+-h<>!QPa&Eft zNkdNWgZ&G!ViWd1{Q||G9MX0KahjES78ht8JhF{t!KD6t7q;EWd(Om9c4ZCA3(g&y?u7>DTnUb{WYp+ot3CaM1-_De=tD;TN8UBwu>g-hC*Ock(~hN0*wt zt#-M3Y_AqKh?#PcpLdB_mAXof*(9AW!Y()eJXyrz*Jq<}=~;itqMLKquQ>GNjInk4 zI*yRyM~72Psx?xrm_lEQX>JV7mw&SCoPMCxpOTATq-@P53xDA_ejB<;y>RNg znPPlz7k|>0t?UnW^jySqYkA0fp_R;R|3Bynmf00)oSS~bLm0UP-Lze2utYvnY?s+vdcdu+z=(nYzxqBN|%zC?HlF~UXmAKFbkMi7Y zS3e|gF1+f%+rLXwVEM(;oX1U9PFXmX@QUQNur&7OIW?_mf4Gp_TX?GbZ8!N(YIn{) zUK7MQU22sw&&q@QJr7%#xc?0*y7;zZ#q}3~;?vSB>_eU!9CGLlQ8fv+Y zoX+i?(0Q4)HRzm{;JI#fZpof`^E96DTwhwTiaB!P%~|Tx3alRO?6TtLIV;evTr3&( z$t9JfApu6(sQL!>Lu1zwBzOGI?y-?=S9bI$x0^>(_I(FXnnqJGt^8IZu z7nr9Mu5{?rLjkeO7wWCun<_s~b=~0UUUqwx{AS4juLw!r=Ps!_=BIA1I8@5{dtoZ8 zQME`x!<+8yTUOVdnf9)!!L>Y8`p!t};V>x2J$#JuRs*qr7b_o8_D>}B%H zS~MKjJuGUFn-ISw)92HnrSWFV!{byccvgIHINv+Py5WlFihHa9!jd;!^q$XlGYr)@ zxtk%tXX^2lB~LpQ^wm3VtvLAl^~IlWvYW4$H@sMKeCn}{mS-ykH+FsU?dZ*|I&`3O zqQvau$|t<%%Pp6F;@aRLZPPPVf78tqJ0}bA|5cLZpAyjbNMQTYr}0)Zo}AuavQ2Y( z{Y!;zrc;K?KD(NyTrBe8%Z*medAaT*-<1qyMkVcaJ2vV0w4d+Qe!Z#tw8Op^0q-|$ z*zWtJdz0Pf$(Adtysz)~6brsLb26`swZZh_i8{*)7PC*%mhSNC?zyUYHAVRJv-$%I z^pCsAd+k27v+EtJW;V}pHl@OYUtD`07WjoEe(q0JzG~uG@-j8vrL@^+-l2_-9@ih8 zo1LR@s(t4twR00sMrbO&Iq}MovGWRp%T*hrW3erP-Ob$UenIc8JEY1T)vXvp`q{Xm z3o>n<2Uboll>EGGO@i2!jlJ{oCq0$YvJFmr8}1(z@LhGn)uo9Wz7^KqJW)2~;T-vy zUb7o=iYufKX8CGX>9ismuaw`p zn{UbNsy2~{IdO|8h*3}g$KS^RA%qvucdwF?7hD1C&3rPO5QT~bscezeYHz(QJ(F(pIV!3KG^sP zyq$A^%U*PDNnt|Xvlp?SMW3lnENAY#Un(?V?aCD|Zq5Ghaoe%_I>XG~Q)ky4uQJ!l zR|s2NWURgNUhgUXj+-mqJSZ~PGT*%FbzVr`CaynewXA2v^q!Jda$CG< z^Vz#A?wnbq-xK|0rRG^3*~2PpM30+iADPdwC$izE%Y(OEf6H0!iL{6uTlM*VLEv2O z@a+x0)2B4u@(|_cUH5tWQ-;_RHYaaJvgDpvYWCj2#R3cNs%I{3RxY~JTGN#A_{*yB`(YcUrsn4C}T4+b_0RzsU7U zeOIn?D{tZs@AfzPPhzc7R-dxTUwp|;yv<+o-q#IY?dN(!b{EIn9^3Xvf%(CvxVKWZ z9o*6D%cNbK8l`P2A9~n#73RF!H+yF8E3qlfEsxW#Zgno#x%E!m;i!jQepskAOLFzD zlFabEea+lT+aK@9NQoA4|CN3J;?&|kF0Zq*cJqge%gXQN`*QwkQG2aIUU^#cg?0P; z!#n@6Dfz}+;p+^wyUMrmY=&3JSp#kVuS%bV$}=bUic6fDyf$I;tG$z|JS+5$9bB4O zGIe21d*0k@tWMiar|W5C^0!w7zGT|fbBl9P*v#@5*RIIwF37j4Ryz8Y_qmqV1^+_| zW|wL&U!QEWZBkaJnbrO((XVQE4Ou$`*f!mHs^uuRD~9*hxrHYUUS@p0XBOn@p>gij z@r9xF%vD?_p>z74FPK;q%fCD7udH}I*W{Ep-(VlQ76=WgoD{mk`FVXxH#eI{EByOQKIj_k&$#Vjm6y?V?2l5rdx72#o#V$?4}L7@ zn6mD_W_V;@?D2k`lQu8(b+6wpG_d@l_tsFgQJU+9ePI=!0nagV&1kV ziXJOf`xy&lSUR>uUktq9R+`di&9#qNvPe``@_l&cgu8(s*8MkBo&H>WuEMP+`=7br z%xc`rq$jv3TKv^j*Eb7pulpIur}zGs{PLFxXZwpbyuJQMgFPkY^WvPB&$tvfcD=In z$Wq?Pwd;j{&eZ!2ObaK-K5V;hP-XQbey5nmsX~p@%~LE~lcv8YetVvAQ7wOt@m~LV z+4X5%a+@*&R<^4y+tU_xO+lMo+~D`;TYR(wR zCT8BL722$gAjJ5TnQFXs#M@-W983wxhk*k9OwYG>CgFOJpOcP$0WS6r!8j+}Bo zKqG8(`D0U)jY|&Cn!K?8T-=Kv1sVIBUdX?k@S?2u?_14W1*3AIU#}8gq-~7uDeWzM zpY|};r;_*i`?Rg0KDDNm<(ofs&zA8Mu$^h4KRqYf)mLrboRxXggp}A1Po8fkU^=Tt zZ~fWGrLoFUyCVhJ>$};FEU&yx-kbGkLf(OgT}Pu++e}|vRlIxm;NC`#+=HoZ6+e#H z37+p)TKafFcYBidsztjZC!Lt;QSfh9Wd4q4M<4Yj2k)soBBG{Rx|zpd>gu_7)MTrJ zK^6Skw8H3;oxgwmaj`RvU#xytOvA=w(w+A@it_FAN}eCNqdsxgVFjP=r}Ijrqw|&? zn6iu4WoL9r3geHB~u`Bgju=(yZ zmb|Gij+_7A7|LY(l5bbWR0Ze5w>z#diHF~e=UATT^|8Zx{V_iFsgH#VI;cZGA& zpRS6hJ96$e`EjoMT;y=zg>dZM#|-!1U(q$^?L){n`&;{BEl@Y_OZtBn)36`Og5e>%VK%8hs{$CAb0r$}qsKPy`KX7#hTL7Go<(*vR| zMC`08_^AlL?iZEL-@p{hnH^ z?#cFB-K=X=w3XjIYKIwj=2Bi4sV^IT`U||ZTyRpQ;?f~Yu_i3uV-d@@896@ zc-tDyHQ&;wT@-lj93z}OH@3ne{QNzozpK_4zi4Au3b0AvJaaeOjqpvL*7@!y)@^*g zF>0xW`XkSEJ4@MP>uxJfd0ZU!G@xYBrB%xQt-qaTedjH_-mtpnf{8%jj3n*Wpoft> z{oQ}^n1htJeLA?CcaGG?uWm1|hp0!3O*5@;c#*h1{C)do!SM64^Y_SotFo<3oc!+Y z#+>{kTm5p{Z@2!a_{;eA#I_5eGvc1^`55B$Xj|>u-cO;bX`eGvb(if9+*_43*ZJ{Z z_7ZuE9S45DJ{!xL$-OPgS1z=oK4nxlT7 zxz~6r>qF@CLav#18Yb6y%;ld(Jnr1}bHkC{T_+rL!kw6iLo&=9ZLO`e1ck^NT>A$#yqO=6GzI{iWf-#lG&kY`()wAKkbZYI~~K zSM^pw;QQc`pw)@#4PE(Jy|nrp?Qf?->LOU0@b|NCM6$x zx{bf%)wzR5VnsM|6;>Twwa(%^^P(2fPRpd*YzyaPuUIv!Y{E17-aj9Cf2oVS{NZkO z{n+=p+582$m-aJ7oj3lQ{bBZ@SMA?_AM%>AGoCkbvTw%|%}-N)zs}pT zW!{&KPv88N=ywh*+VVqEHd;iu=+TPfvYU)*n0NkqEjM9B)vXWz3T}j-)Y59Y_2AVf z)|>i0CpTX_TOlDKK54I)Te?!(gItA85AEX)Yvo)jJi&ZgHhh@6d&e-U;0*UdgJ7Z^;sK%)7)H{%R%ZGhze$TU^{Ou&+MA``vFgd1%)Kuj zFX%qzy645?1-GBAo+>cce@W>f)$sE_4o2H-{{HG?#v|AHpS35o&$~Y5gTHu3DF3g6 zdF8S4yPv;Z$1a;Twd9CKN{W7aWc=?s?UEU-d*1I^zuAjVJNb9Ri>eFS?_aNBS}pqW zMxt2IetB-$G)4XMN5xiaJWuzXQmoOwP-caMY}!iG8u@jXN&;RMFU^@7`awrw(%Qpo zCOvcayeYCV(&V+*&AinscFJD&o?@o=VZn}yOUt-zR&;V!@O%83tbb(2l4Tw#$0dV* z%qmZGne$ZoMUY&EZc&N-)xY0^W}IM^lCZKryK82|ac+MnpU9hq9$bwHg2a=KT1Y{{%`y{sd8 z=*u*@m{SV_Cv#7EYn)u;Evj&dSv9y#IwDtXQDAW161jS>)k0O#uD$sSdrxE*zvw?c z|HtbYX&;4m8SGqtMJzgm@B4d)wkX$L4V6oYrWW(%Q?4{l~ZJFc0KWt<`j4u-2Zf@_383GMgm0@KdNEb-y#KCW<*b)>XZin^+hyErU(P?%Gk(kOVwS2|C(qs} z5Cu(ecR%@>vU~HXjz1~C*%X60k|)J7pM85}=0ux~&$hkpKQZmWwAq=YhF}D~*w1m;a4pv(he!ho<~x30yT{3zKxfxz2mxEwxG!!s;D;D}_pbifwe13Y?xV zxO>OC`MXtUx`$QS>4ELYx?<;^~qGiPk|CU(A=Q=7Ku?{pha zhnbT5pX^Osw9Eh1tlQb21q`qEdBsN0E3lt%eDa=NulUa=-kj(8&U{{Z^UCwz4=)rc z@#hef|0`(r^=F;K`Zitdj^%zEyQ*4}*XVIa&-qd3FunEcBB4XCHf=CJ#1bthylvs? zdv8v!K7HtBM@r*voo%5{bemO{GfWgb<|DmO@Ni@+&v8YEl!&Q&&bBO<%(?02u;o!E z+jEyEx!;`CPwdF%UozisuWr6?(7!)>nw{%{{@FaaW@$8?e~!YZN$PzBS!=yuxD*}f+?w==Kbqswo7()9YN{2kU88n0AkxQK!GKHO z#|un`gGgT=1rsj)U`K_LqS90@eJ5W9eZTyo+{7Hv;vbmtpruqO#)H;bAuWdj*=fL~ z@03`a3X)OK4=GAbDoxHvE%6QJ(s#{E&QHnAOIOhMOi9fv$t$l8YplI2L6l=Jp@U?Buo-QDrpg1++(s%ZB3CVZ&bn#6rK*S$dIJmO7BsJGFFD)OmtSu-tJ+ru^ zs8T`0F(p4KRg+8KAGA0Q6zv+YXx9WKnSz3x)LhU?Km$;`>W71t&qbRW7$|_Cv4NqU zk(s4wtb(zbLbR!crJjX}G05GfmKJ)3CMKo|W`@RkMiz#~u?i+83eiU97J9};2B2hO zpkQiXre|zo4hkNSS*E7udS-^^pujOuFtISuvotkVFtsqyGc_|aiB&MOP>41)GSxG- zG%!;DsWmjRuv9RyG}SW%X@ZNGnwaXD7#bKU7@Jw@nSs=p8W`$Xni?A_n3|aB8C#f{ zD;Qar>KU4vft+D#V4`PiX=18iWND&jX=G@rU}0>aXJBrDq}4bEBsexPA{`OkT;R2kZkai$MhZrt zToVKl@k`82Ma!RVIf?1Tpi;mQlp{+NER8MnEX_0DnYiT;e zCw+#D{FqxCR{2)Bd=_HZ_%Qm_q^G^Smi8&zn0Q>}KDJF;I(K#4{BLngea{u`w`|!V z-E&>*_=NB5E!rDye)&~;ox{Vv-(Xqg$ycs3@7~O3xG|mKhM8_wcHFPuDx0|yYNQ>0 zr9VF9_f@L$lV5}4YKDyH$Cuj7e6JnPWw4qPe~ZhftR?JxEc@#o*8dF=tT&#oY};$d z^Zw`cSy%t9`gi%1-%qieR~Ggx_a^3ldCII2|7-Q1)l=p&D<9-9-6U5N*1+>rfss{s z$zh#0*4lrCOI=+Y<{F;c%&}Ic-t~Q3kRkUZ0kaPGggsw0a+Fj9x{~$T6`7B(bNeH5 z=r2cp!f#d<^+Sp+g&f|3NdkEeGbCH2T8xgJzvo(|J-aH3~fFY(QuHB3{iQ25h&mpe@r>NW~BEk6V|cu1c3 zuG5lvuhC$ma3Z&pvB|@#_LqV6_Kz%g{L4S0^S?hyY^8wv`HXtWZrdzgg}M$7R8F55GCxHM;w;HvOQK-0~OOzvw7t@GEz# zKfAo6-Mu;6FKOTLTCJ9u|C3`L?Eas&>tR*GKJi283A48_R?evLu|LD^ERj6<=yBB_ z3mf8#e>%;0+4obqJF;iCLY~Y7&g2=#1rOUFGLhqdv0l+;!E(l0N4fL5GoOeV3g;zl zy7c~`uIAbOvBJ;q`_+HRpC0Faqq#ur_RoLo8s{XtYB)!*_+GM@vX%YoWd6O6_TG5T z=vrkoA^S-4hK}2_K9(z=X-j;xc!sc<^CUI4QvvyGzE&lV@91x~yAwS1gUz%3t{Wtu z@Al4d{GB+zLODk)R-pZxPV(bJxrJm-v) z-`e)XbLb!T_!;UkL3`qJolk#6E8jI2?At%xQ83@RhWn7gy2%!1)917p&gHcXvi+vG zp)R*%x z{eXp`*s=$5TCUd5p5)!R%)H`t)fw6P)~rP9lSK`Em_}k@)6U# z{_2+E02LdC$AYs>b~45m{`F~DDkc0S!%gY0E-w`>&8*;i6!pmIKkGKL z`O8I?T{yX7_sM_fUvmA=p0zt+#xkzTiS4H=_#XGJ?cVk1Q(My0Pd9ei#5GxnOt(MC zda&Z9!_7k7Gxqb7>+D;9cpf>g^w0QFvc&o9hYqJH&zDa7edO)J z88RJ?+P9SdU#i%nqnFQfEV;-)&RDeSJ@5VtW>2u$#s2BY<2Q3Y%ol!Bpl-c-ivB~fUOuS{aj`|B>-u*Z&EOYT zTH{g?Q6v9uf&779o(cC_cl6E`T=Ru9%r`8IKQYjh#gSROTc`1J%&&uvj#ErFExP)W zH}Rp{3a7$P=Tt#*H5)E;4@w*qS+s7zoX0}t@bo&;) zeOV_jrJlYWwxztJZ0(!4t-oHGX0PA*ZCB~_`vs4G&91w;`QfwP?RTnfFFPBZxBv0E z`2T6Ix9xuS@2&0kfA9AFmai!I?aXig=|$h%e%pUXCi~m|?Ma{h{pQ{M_cw1Zzh7Km z@vrr-{=P4tD)~#rER79QoAv2_C|xoyT0flTsc{IP&Aj zmLr#r{BkM3)a>c6BI{SB{qjm=RhXUs-@34chnaVLv#escZr7E){`gjNPX1089`WVf z+(&djzJ9d-_*UgB8KFt>H{88T;^KfKk?e!Xq;rOHR&fA44b z5PgvEfid%!_iw*{d!O{|ef-`1tM>1%PyIhdU+$06^NI_aefLXhUU$~Z?|*ruT>YF} zeCZ+Pf~3cVpF1b2UlIN|U;p2qx5A%3CeP3P`)2p~4PF0p*qxrIYul~-W zPrmi<|C@4CUC*rli&FW%&^>zRxfqroPq-5IZQ<>gi%Y)ySCz-zGd3u4XQ)d{k&Qjy zss2zgg5P}N#SNn84y;cQzQeS?`S^oG3wG&FCi6pc3&i#{-+my!q4C87;~!%0nvXwx z|H1ZN|F*-yAEabjmM1Ej91_cDx_*$i{ei(A*8Bs46~g;kTtDdT;eCI=_yW)Ww%ZSx zEBNoVi~W$U;Wa;;l+n66K~twO^#g-WyXXbBw-*>=FK}uvl-}obHima~B5w?H_Cj&F zme&uW_8g7jw|A1aaqpYd-!V_ZLxNMCN$}+HClWtfYP?hyDY_*or72B5f8zNQ3roj; zPWu$ZgdEqXOq!!o&no}pq-N2qIg0U<)}Oe4;`);Q8s~kg=_kvdRDUY}#Qc+e)-&f% z{-5+eDgQLLbU!zlTcyioa@`|kGa=`;2@yZIYaH7qS@ei0d{k)^@>=QOsujpPwbSd( zgT`2ui54rIwm#vSuR8_V#^`VAKIR(hffqta^p4WX$w|14K&_*fy1b&M`f{x z)z(WMQm%_v_8vPR@41Ya8|W%G%g9rMp9>IFGJn^c)%w^EhcLw2P) zZ;E23itiC$v59Gym~MM$NBXR5sy-pKX^M{WX;0}$$6KE4FI8ewR9~vCK9Tj|(B={e zS-$E7{XI_Vhw^^d?%^^&Ait3PS0>vX-unmUKM?$3yoW_!Sa%Nlo~72u&**HQ?yIXi zGbkp+qHcj{Az4@-NU zJ|63}l`Ec=K4b9-<2SD+o88%T+xSpm$Q#e%2PsdlGII;`Pf00SA^o3GW6$FEuf40K znrvsOT()nRnA6Rib?Bo2uZFZ&$M4OzXX)*`5neJS${^)L_OB44mlrz70>WabLr=I0oD%bR#WMq32Fq1bbea8>& zgV$}YIq_dHXS@GtiMY7o&MS|4zq&Fq-i+ycxvqA?Tl;<4Tcm#HnDw+tskN)G66l>2 zv*+T~Ehi=i$lq@Hx6`Ba=q=lGQX%oZho7!X3@e&pbGYwuQQ1DFdhYLKlI0yU-^Ta_ zg^2fxX3f~T!Fg^I|M4eGG6oBGR!oj8DQbLmC`K&laA=cE&#n}?91hRcr8yo6>&2!W zyOsT>@qE2vP)>lF=E=o8vrf0jG8?g8`{{a->5k8W#H@|GGbS`l_t9cyiGVDJ3N|$)IRs|09z<-H{iURW0x;wY<|O#r|u;=2J(%9V$|WOT&UsI*Xb21{-|Nxnps3q2JAj{Tmlv-=DY8 z+0@ePotFOn8(}f$r9_eS-=?N%Tf++fptz1Jr}wzG+pS~z;UqmLd%7Y z4FU@p7s@T~#hOH%4#Zj#2?+b1p*ymxAAtCW`T`g+ZH%A>W(8aH=0EL6$Ybq=q1 zZv5oKxnzxt*3%VtZ*Nn)`Z`JDpmqDgiuXBfkM?Yqka{)uWWbtoHfQHOS9y+BEH_U1 z9A+r;cJ8r&DcdJ4^vGv*4tqb9$F<_L9*nxi)L|jE#*F`Kq(8-{1dl z=2!i7_1Sy3?0oufT6$E=uWQ@Fc2&yVw3)u;tm5p|@77l>dv$cx_Ek&HUVcCS-U zZCq%$)$(>mt@_IM3c0QH6DTaP(i@jHLSW3vS%Ms{PIWwa9PlsH6Y3 zExK`klIZW*Qb+4tg5KnJbp0)L+$g`S>#yme8~lw)r^NW|&1sv;QwCD~tK})1UR9{g0Am z8+YH*Y^fD!P7u9(uA0FoV{t~_?o6f}XX8)Hx-P_klcJMd8oBl5S|HtP7 z_ty43=*=~6$!q?-@x8#EuJRSt8~!KpKZt!>crWje^cCI{8$~OfzSu^z^S@dynZk1~ zTTV^>fJoNi-^b+{C;XHC)4Ruc-Qn9#Eceu-PH^n#eKtWVagmDRUg>EMqq$E=KjHnT zaqrQzhtlmvy7i-8xz3lPw^e>mW7*%oVZDK0#3%1~L;o5tuL`&{qMeYutzWDQl?{of{)y`k|q<-K3&wuB?|F0iE%_LDaQelm-cmsoxiuK|p z)vfnCzn>D|-Fbuc(_WU@3@`uwtyS5(PAo9Zt>tE(-=^7bU$eZ3cR$E(!TRN~<$-6y zj7RQX5}f1fF7mvD_w41VuVQwI-PM}wQnb}Y07f-wQ4GbnH)a4QOS2waf;TB6sMzRp6|ZSFI_h4?5v{5tFJ%aT-G@)NJYz= z=c$)!(Dp6Px3uR@+oGVk_uu1l9>;h>E(zvpE}f*2JX0lbhDG6`o0I0>n?67P5cgE$ znte_!K}%gDmvSsv^J4YtwX1u-YTS%C_h4Ii&8_Lv(jLv-I_=MllG!WyXFvN>Ge>Fu z_iLNfXM1>;aJ_R2Wbrf;6*j=^~y*F@ zRj9RTOtxsi{Mu!9mnWM1S)F|A#H!V%>FgqhOPzh>mn{AIaPhit4grZXC2Jh~%2a|j zU!62}s`T28s}sx`ZM|jUBmO@-9bfnR(CpLGZQuSrR(@!E`<2Nr@|Q}_pYiDSTkG<3 z-*3s5+q=F~Huli`xaj^vOPzbIrn8vDZoKo^)+#8z=1|?y9S1#nkLrHXnrUSE?Sx9G z+NQ>p>~mkTr>g`{T%$e7)5KGeN&3>1$#MTH)5W9}1h)UOqJEyKppW1uC{OrFU$FhIlKg^!}?PKrtTW_CfZ@L-R zX{lv>I?3#&@Ja4{{U;`5zEJDdlhMfe^EvYViG`Q!=DruZ|ByRf<>encSxNVth=+@( zzd3yTYlI|c!1FUxR3~W(Ena_q`OeD|NXPF zH(&R^ypp}uV`|3&0W~|rzmumPdGYye?N*J7Ct@dvF-+ScS>y%3`UwbTF{QG`F+03V#FC_nsf8ily=%7&4$}6$+ zHGI!+`)h3q{M~jm3l8P5iuwZEudEPpCUIEiX z&AUnqEkC~dv93ZH~cwwgDnOi6G;_;UXa@yT9m^|HH5r-p8s zRqV+*(|MMsd4GbWN6?p$%@(Gf*>W9Tcb-j*@-DIC-{W9l6kc=mi=5qzsfrI)@7x*q zqBpt7O?~+i74_#ma+BwZ>Z&K#WGSr;47_+TFmPhn#fcWhS@-%*#?D@LcYjg#_nEf! z>!qjv`Y^>c_V8)z@Z8w9zptm)YrT=3wtL;iH@iOF(bWHHQ&V#5a$&O7^QUL$PnaJ6 z=~Zn19H-dG3+Kug$)7a1H*->xqNdPKKc#z~T|pX*r{+#oGfEEL8KSc{!@;I?M)9|~ zMsrv3MxB|L+`q(M?o+SO`squGpQ~nm&-=1O>;Lsbo*M;{6gySkH>>UHz2T+(<}cxKZ2Gm|+yY$ki`@%NI`7Et7z`(O3>7lRbuBagIfL{{piWo-51Tp!kB zmb}W4KfFxxz0o_9Z&Lh{{Zi*8coVivIeNZ#xl_$ozop>|^;!=kJ(efwN2Q+a0DYhB9e?ep;wWAbaToHZ}!|Lv{6 zJ?<=L4WIY%T6k||;koJW&%Jr$F}wQcG~4xgk^9m<-{a+9n)h4s@8-TN zoqY27wfNoTcg`s7IsSjEh0OahwT@=F<&}N&0-{qp|2f<0Ud%uI_sg&IZ_=bvTa0IA zhUN=u>xdfN%LvNg|0_O~snCAT&8DDtnp3L1Y6?{(<*d!N>4bHx;p<-;)?pX3{zOHQ zFV8BC8ND;5e);GquYXWDU&Y!$b;DbyC7c1AO*f@_nD3ZMhDB_*|7N~)zQE@9v)8ZN z6TDsXWxsWN<&{ET*XELS|6?w(+`pY(WB;S?3HRaO+h;AaE6+JE&8j2sQw&XX1uE$Uj_^6=TlXGWV9cC?5!x`lE- z%gS9j?Lxua1=mj{GBz*jH=fLQFG=c_$oX4L!eJ{tEqCb7GM2h9_u?)_cC(wCk3Hd9 zEEQkzRwj_&&Z_$Vzf0Ng>Kd=~A9N`SN4k$?%(c7biUm z*pkAdGRgB?Sh$H`RqCd|e{28j_%o*_;$OtI3zkBOM<262)k|Hjv{b9oYvrRLy|S-b z?Nb(aF+NL3TF7;O^P%3=Dk0O32utNAcP^i)K38_z-fhihYu8?{pR~GP@Eq5%9OaKw zmux!j^2ukZnz)$SqR!Wsk6cYJ`f}p-6WJFZ8-ptPRDyO^?Jx?{3}#8}yR@c6yJ?M! zw#ej@)#q&X?NQvPRZzF#vNO}8lxfN?MZAj6tJdD>yOyRSd3sIp+K^RdFV@~U9`JtI zmVLJzE**4f-WYQH&)PiK1rHJhziB4DscBs-+E~R=dp@t1Z^zsoF)z<^hEF#Yv-OKj z(#THv{bEXr;hRIP&njojM@5+?#r`E&+o3= zHQRl~;rx5g)Niqy#qa-i!sSv_?9H2jhLexV%{tp(C&|e@r!ZDb$>rHX=AB!b!sPQN zd<|-iS-4)qRKVcNul>b*4?{OkU3qxH+G8rYORfsC%?%W0T=*nPz^bAknspiTGUl!9 z6_->R4Z6WVVEpU=44kNBNvV6u2G6Dak#E3-3FRN2WZDo<)@-X+%+3ud)NnM_@oS(&*sbKRC{ zTS7CXFFi3S6VgpOwCrflS{W-wnW>43wRIma=DaCT9De59jC&G=&2RN?`$ez!^)Y?& z@NJhU`xL!%d#on!D&AeRG9joqxZOqlhEnlr;W<%rm)RuF-sF69HOK8Q)?cc>GA%h5 zFe&ZDMxMIN?Ke-oemH$jTAU>J)x}1umEBukn@(J~c=>;Z-LiYH_p{aUXa8ZIcHyYT zguIG|Z+owoGt6hd=e_zZ_uW}D%w3+a8R@RNII%<2J#W*@)duEKZv7WdY-*FY{-cq! z)!m45neMc?uV>5e{rc;^&H8sf_LWz+?63Lyq}H@F`}T6Vx*z4v;yOOJjUVlc-B-W& zUQt(wN!;dLH}CE{R{mz)Q`5=$0rStyl-)QZqfBo8m3yYU-@SV~e}<1!(4P%0x%+>8 zooD2^XY0zh=g!Xc-M&cvYr@{&g1-V;w!E3>dr?ituj;S8)!~ZXXhFeqiU7O5J9rCIM)lw4a{_yuvtxDQbb~hDl2~~8RnI295~Yu4M6G&$UE1M(^X#81 z6VEO*=Il>UEPEPybi2yi`}6n?4!Y+0dd3&k zU6#AjVz>KEG`%=Wq<6tW(Z=q_?Jg`yk9Qh39rqO3Z=3bqi(U23N+XxV%7`l=r>mF_ zvPcPS{m)=MSvkO<wm|z?x#)CduMy!YWK%Nj*LLzcJ%-znUYnX0`|o?QJSqnm*JcbC5jFTdv8`Lt=JlbQSM z;4{n5D$SU=>v$pi!}iIMy>hdROv+a{-UBm_lI{!$mSgjxSDzBX~GINrK$N#)_+rP zlsdbvH!acQ^DEEUlfK^%*zlk>$bY(hVR7oe*2zh~j{UAN{vGW+wT18L<;N$tT$bDz zuB4(rYlik>)19+~d^fvY*|6JEzvISC`*v~Hwo`I9*sK%uWm+yT;Ees~%8;k|Al=?Y zSF9u2yX9FyQ@PKnyo2uw7V0GB9eKo|xF|eb(0*(&zrno5=J|FE=S_fr;Fg zVpe_eof|8<(oCluJ)@P}ksunvcdy%Ya>u{N!FKmsAJ>09tSD4|+3fyF>30jA>r`g$ zOWw!5ukQZu6cNVzmloG6%)a+{>i+zF7EIssj#%Hk>v#F3g-~eZqTkc|roGtqa>csq z^z~=U@9PqSu7no}V~twDf3LkeKtD ztBHnJjX1?k600KRN_bc7t`Lqm=>O$FYQtUD8J5b&H}FO?hc{|`^LBWk7OJg6I3LoGY~3{bgNt=ldn^XUo^DPv=nnwf#uiB;ED)^Tq!hZMoX_ zFyQ$XO|{vYS4&Q)$npF)nA4WMMRZn|SkHyH|K_`!9jgw`Jh1qRg~k00hf5c;niowh zZF{h=T18tTR_Ef!KY|*rPLJF#D+{ab+Wz+b{UVj!aS>C$-Rb?iU-a_)ii<)!W?tad zm@ND6$(s0eT(1z_l)B5 z)9c0KBMlOxe82sQpI-UQ%6N9M+LQYmb51+!O!rBSniM0ka`oKHQ#qnpe0$X*s%CG~ zbh#*WA;I)y{fY}=EKHx=)R*sBY$<~mq+`%<}fV$Un$RBn!}p0$ILl$g4hP5*>X#cI@$L_D;VXH78S7?E!=D0XME@lhy(w zeGl#lK2gnEEIQnss){FEa}c>HxJ0x=U7saW_egN=glnDglan2z_$6mu*u|#7duf+r z^+jXLRORKo&A$#y>Z%vqI`U-7WlPz{Ev|CsGhZwG^VoNvLDqDIecOKP-pdG&jw_km`_H3G;>DIy-vu4h>cUzm0 zn-hO;V@=@ls2zLu=2liar@#J`y7-Ex_p}vrHaq8*m+!b8zH7GI*_?9yH(5D_<*P)E zSKU1u>n(Llf1j;^#qGQI8iQBNT)b%$o0WE&gQ`w@Sp*mFpZb{i-nFe8X0M)QOP#8H`zjU+HZv71maw7sN1O9aJHVf-B!HCS=%JJ?*)r!&_cmB#d9q+ zfBGC(F}X`mOt&mi(0LToS~Jz`U(SJfVN0eR)R39k@>W75taonVq=TZJctu{p-f< z))lLpgY(uMbhY38O-XNU*IG82$g@ACe|F{Vi?fzb42$o0uj}~2WnG~CYU`=LPlVR+ zPAx8DdHT+wc1v7R{5GS@w}bwLao>M<{^Xgc(>vcN>Md)`Iq^XG?v3lGuBTgNJh0}9 z*gf;}x<`NCdH+#=a8xUPueOK&qupKh_a)v9%_S#AEhf+Y?R`S+?j7!se|x&tpSkCI>sT49#56Kn<`aOPA)yTcc)>KbVZAK(M{!MkZgMDNn?n)PG1m-S^~7QGAisM-73;M}0QN=m;KvY%IT1dPl$M$IUrHMO!}a z-MsyuxOVr=qnOLa!ON?m>n!md-e_cLf^v3aY~bmv!v+F-UTc4wA=h-@ll$0(UA#Xo zu$TT~?AzkUx>YCT=Y#$0mz2h=a1b)yZ5;L1t@QI9jZDQ~2Fm#to%E*~SoCgw$RWPR z>*yqrmM3Md51%+Mk9>0Etx=?m)YbkqI*g0_#pkYxO!%&GjQifk7k4>Kf?^ZDhi>QF zzWJKH#ez)&n^!S4?k%o*`z6|3!NqY~eoMFa?l*Iv=|A3+(PHwTW!1yQ2RzGVe3!2~ zaq7leu2YkvMV(du9pw&+t@?L&=7hOB-&puPKDF3{U&Ei{YKt7lT#l=ZsY|pUeK_@M z`_BBMOV~HrPcl>1ohUhxbz)`R=I_@VwNAw6X*Skyr34C8%CAe_73PrtOaJ%9oWD;t z8?Kw0Shui!Uv3^v;T|(HQ!uksfZJ|vpa5Rd4i|tkV2*_gz!^qx+dzxqVLX^WVihb56)a2;R#=!S zM8m@#tOFV>a05YtP|w9GSQ;uAf|j5gfDXnr0&TVc2XicFxw%5LnSqI(fq}6l$_jIE zLc~a~h?ELRxsVm+#*lMi!6_QD!W^Dx-OyK;8<^@DnH!rRuP!$<)Uz}+F@!BIw=mbU zurx5^LRwyKZfL4!VPs&;rH^}+xtW=Xo~fyUg@UfJk%^wUfw_W_ff00nfT4l0xt^hc zr3vynbFizSK7_3^H!(2>t$fE@9GV!Kn{go@X8UGuWliwaZ{~Zx75F`SqH^|%S?;7| zo(ofRP6w^kuE`|rEYEAK!5`}psK!{67~@@KE#`t8>9C5#*l5(}6lOlDhDJ~<+P z-TxQcf|b$^@#V)ad{n-#w3v%g?l$9uEGhdh?>cw&_x3XioMFHJR=AB{wQ_GZ;~{gV z8{z$O_0R8C&z~)H?9Vlcc+09UCl+q|crARx8&(%JH}^NXJ9bDMz9F|oru4V%_NlU7 z*BHM1=3H>M@wQr@RDJlh)B71sWOx-^+x&N5-d^%(`DKP%3H3#rA3kpPli9lSw-Uqg z=?tFSjy-ehK3%Te?5``Ir1{5p_A1tomG5q^$#L9%C*bS9rM#t!rEkz#?cKW~_SrvTStMY{FmaN{ z>pqFD7ut^^%4VJvMD7HeW8ukr z=gKUYs?l6J?bP}MMn3gYpWBmfZT9ds*vECad1KsWmx$$&3@4kEQhf3|-k$M3W8S@& zRrlwV8`mwS2jzTDJCgHevG}EP9qS_0I5XLg$b6DJqx$~nyu&q141|BrSH2`TSLlmd zPP*7L@x;A8J?D=UaP>L6WjX#3p&n2^b+HLtr$z-X$W3na3 z<-mPbiNB2n&L_=I52=)3{JY`5n#boKnW1fI%FmMHpTGL4QqlZTa>u&b4eO&88b<8b zU&Ou6oAptdq1>DOD+3F;S0;wv`GJ)TcBLc()RK5 zg-+8JwIE}k!#@*la{Lwil~L{}w=UJcM}L2tp6~@3mE@(d7U#E~IN$t%=TnK0y@&A+ zMy9w7Wv0weL|6;ad+0oA|qC1S<*PH+S&+=u$7O~uCw{F-kZQdwx?s?!s z;aT#Y#x3jTB~^(oVw$-2tm5e{k3F_7v)3?sm)pT{$Yr7uv#Byw|?T~ z!s{oNe~j9>XWFW2?pxohY>yV&#Msv-%D46kn+4slOiX`uI^KKzig?d>M@zA?3V9x{ z7o~1Y=8q-zd%m!{l^FBBapR0Tmft4rvE0^nGb#H?3{QRMge3ikd*1KcvwYf-MV)`Q zB>s()@@IGcYp3<};ExvD=(V?I$3G}6m6u)6=dYgM@G^*Zo%Jn;(mmS3Rvmwi6!t%U zdg8NsqTM;=nP(kqkCYrqI4E4I%Wx+tUZ7ejx~WF2`XbYelYNW!E4+Hv7} zFS9A{_Yg_6Jt0$p&&FhSFkNcXvWm+?~@NE%hUuan`dD_CE|7qAN z8;--?t71BKJ-(pt6qnJ~%$B;1eVX9gwL!024o-V;JZq|WR_zzpmKx5ugxQ;zf35g; zq<@3cD}Ls{_OMK?gsFGrOV2T8Z|D`@5La&~wr2XGch|iih+SE~Gh_C%MQ0g9|FIv< zx)C+!{N0S}f@_#6w}{;dV*OMw$LhPEO|uGk;e+}#2i5b9NtH4> z2ez;Z-ST`>F1O6K<=1VV((j3D-uvFjJ+M#ySV`b^j~_;d+y#Cu;&h%Av+x(=fCvCqF0 zK7;XT*qep7fAB~gW_Qrb<~w3-fZ)1x1|K7>9#x-m{J{jbFT zw3=5ouDc`rr#$p$oEMpNT;QNZ@3&>#3A#zDwoI~-or%Cv=fi-^eOd9FIn zH4*d6RbArir`M@23E5$_k5!B1L~F#{16!X?Jg>WP|Fl1BPt=)G3pX~{$g*WFd-y9g zvC3*&rLWBIlTV`)-uyS2_n$Mx-XyqE!u`i7%WLb|gPzwnB!>sgU(mVt;s1+2{`oh& zww4WC_UU_iflA5=A3jI6ON<`u8&(!Af>Y49-r++`) zu|akDdd<(%{r)VK{(Us=^Y`3;|2bD!R|~#Zy!Uti-RT>Q4!LO@@{@74U&8VCMDvUJ zcbJSBUtO4X#mse2N7)MbOZo=9*jI$pVc;aYXnX=JM#Mao+gRzhxLkb2R?**+;`f=bNavVYulERa-02m z9LJU`mL>dg@44TnaqvUZ622w90-hYw*W$Gl{!Fa5ZvVI6@BhC)ALj>siEMjpvG>tT z@k_tg+dBU_zNOCgKYP{N$i;Pk|EpIWHEzz%n8RO@_KWTBz7tc^=l_4#6YO7TyEwe* zd;isi{+S|mTRx<={$KN}pL5l``yuvXzgah24}NIP{Xg`_xkL4?ySQG)LyyAuT`ngKg%&UUV*88nZt_5{$eny|+CRB% z6DM~VCTAQw!N9A*STE3IlEI_pSbahE3yX1cZ`03%haS8=*!95q;Ld|T4<2t?-SoOC z^x)frvIlJs@;Co(Zg0Nd%H{kppgo{6qvFN33pz{o7>F0#vnCr>Osulzi2XIF9U-|audQOIIplE^~2QDb8dIf`7LC5{h85ox3**R6t$k8sEl)-P<+#L?Zo1Tv-(Wu zX0u(n6CE76dBd`8->&T{JF%1N;LaP{HD-7xmF=3o@8GOseHZ%!lyx*F_D|YffsAnf zuz58PUawR?HgCuCC*Sg==f+edo$XwDe7aoyyB{_UkM+BDsmB~wmvui~v*+f=4`XUS$)=sb>iW%%2$=*#oP_HwqxU%p(-cUimmz1-c(FJ~9u zIDN6aZ(l*l;fs$J%l2)vpH*q{c*XyWs#EKxcx%RM=Pz3Rec3bDv&OTQoBC(YyP|Vl z=W@-0$n}v+BO8BT=P$D6^k1H{<3jAEPp`gS#!MJye~Vet)lt#!!!T4E%~?q%*(RS zDF;7)5IB>x{ZP;kzO=(mGuY1`IK4qCw#6&uwNC}lxx@Pt7;PAf6U}9si$Cb>VcLI4 zzkqRGbLj`WJ>0JoC3xD88Zdo3yywek%F5hhD`MRz%HXQHAU{G z&$bC`pO^;u`W*4uHu3EdCy^voGYii6$tp?e>nF<|ah#K4SJ}MBvuslA6WvX-cXUXb zIkc!u7Wrg$(>bWKeaXaFY0_UK`O}mmB$p5L$y2_3#S;JTIrIyYGKr>#Z{MPNUfMSYsJi{6;5lf zxJ_O$*JL5zD-~Z)<5!FFX88nXXnnn4GRtr875C@C_GcB&dg}2`u$krAY~r#xgZb8k zw97J9)9rLLCrzD~Y<$DVtov*t|7O-4!Rj8}#M3v8cNFhC5SNyHvuayoZc6kGvu|wG z-Fpw)CHxKuw@+lhnfOig_pG^T_8T_8k@+qDSNw15KQH|Y_g}dFvi{rr&r|=RRMhD& zXAbo2*ex*UO4dK7`ic9GT>mirr*@5Q{PF9L*KKNa|KS$bYW+EFUu*u;^$+bf_vRn3 ztzxfl-hZG2|Lp!ZynibHssD%FKg0Xy`9JvoSpLWOpQZf? z{*U?v)2+_*e^~rS;(wR?bM+sd|4je4)t~nNc=?a@KU@35^$-3(vHvvxN8~?a`{U<7 z*ME%vvHlO!|K1vJzmMI2%Kx$dpB(?lbc5ik4AqnEx*J6A9lLx{?FLWv1NjePXB?xi zO#YgSyl2w+$De=R{ZqKF zLH|j2fqR|H-O2MG`7ieWtW#xn`bW_}hWLZwpF(#8+n;Y|Bsn%+7!NBsRcLJ zI-hDkwBez$+dZ?aFEchi&Xcb{-qJZ)&3|X{t1Y|V?R$Rj_q+NB2VaUB|1mDARer?a6Q&HRU{;rl* zl3s0=dvDX%t*>%#$KU^WY_@gz-mSmi+P>fO``&N)H-A4Yu9bgp&oIB{*Ujhb{IwsS zK2&z^uPyy^r1bvBUGFRImhX9I9rs;!{qMf(wZ~uYdt7?|qy1;u!*6f5ci*}jaATXV zOA}P6y98wB3L1>;HMlGL_x2e(km=(?WI9CSP1|J=1j4 z5lPhxD=aSTyd%yz)xAQa;BjxStBrw(a9iBO0J)EeLC;s8cBy%qWqALp>fwk_+p@jd z9;+Un?k#jQoTIf@`ezsOz7zZW7~@SPuk2Y1VF|+YXbnRdgHWv!FUc|Oy^^UV{ zTv5V&T9brcgvfaEl?W+!Y)JTcL2zQ+UhgjJrN90z?Ui=*IR3CPQIMmgBckYv(uKgc zYD(oj8CUXZl8@vh%*#(+96R;%YEg->dC4+@5{4xOo-Mq(qO}pl8}9oFUf;X?PQB~K z!sz3ZEW@8I+jJ`FyTtsCu&{F9z6GS>fDX5f1Wt9uD14u z^xV)WA;rxbyyws8oDo`6r_6j*X=%+G{YBg*96jq~qWioj2T3;CegCmKDCp4bBz485 zH^OhOS9#DcvXsrMz3fxY7U$IY)n=;gwI{=$o8)Z?+;~iK_r)AfmY!wW%*y)DS1NAZ z!BP6Qc;;r89*qKFE|(+bciNRTIUmf4>b`yAT7cyqQ?bMTng^pM3hq!!R@&9~bgGy1 zM7>w;OB8-}Ox?dQSD$~*-yU4; zm-3dFCvR4rFZ?2IDf-oN`s(TI0h7#EFSm1dk>2&^{346VUGCdE1)kSWwl$h}e{24N z?DvkU`yQS8#FD@J+ioqf^ZB)w`hs05r8At3&^+dMK$a$v2}t z<9c1QmiE{1rwdmcwcexY81QIeeCPe^hD?70jhlb0x7xOPt3zg%N{`NK4Xy=$j!qW3 zxSDI3+M%UC1TWp(5twdLGIi=)=6&a;KAq~kI$88Elbve7&fa&+c5ADPO~20VSDN#C z&lQj7KbM|ZeQ}BJ<8O=`48-R+9}(c$Q<2~GqEn(Nq9vkn!qp?*T1}OFn68{ z?}i6^IW{tF6y7Lb$ftODa>d*OuPto*mhNj>)}eGn;0V_dzay3{ZFQ~Xt#ZP@mfUmQ z!QuGQaF6yE4ZqhK_jJE-wDnz3w%BrhRg~zHykKTaUtz|41Hpo&YD{M7d8U82ygJ`@ zFYDJ^wfMgB)Zb}eiaQiE%D!2e{j>Zbuvg$|M<<7*bGtzahn7IMpthi3OHV0Z;#}v5 zbp1!lM-(&KIFzdtw|sFBEerPj9^PtIaohUk@h`RoioxG6eh>X6SR1}n{!)3!FUCtH zOR8V)4t*Jyk$<4(fE8B|Yi977=`W&Z8|B`MsMfyQ`R3!Jy_$pmrMd6K_Xtn3jsvyJGkQNWu2_Y$y zxa2cM*Dd?EYWAm)pf!_x-?qjn?p+wWj`jEp_r70&YyPhIX8m!WykKy0qRx}k9pZj| z;wH(#Wq~&LmiWGF`DT7;`^HDNcktvN=43g}S|xq-MQ+ZM+?)eV-$Bu@cf`_#M=|*& zkIWCHuXzWngIxrZIFhU$$vB0z7xOtLIwf^TUuiV;<=M5o;Tyle>Or7F`Q_ zKYw2EJ$aj}BF*VG>#O&F2rUkJYTKciy}io*yUm-w3o^Hg1(&s~;+*u>ZtDv(|Lpg_ z_GVp_obfSb>UT%SZB_CgN@S*(-Sa^%Kcwy#5ExwpVrA zY_H;eyfmuADthnP0uIZM31)oT7P`MRnXu06#{N)KN!h$5k*hQ>ez~&c%9AXq%s*1= z)qE$f+L)ai<(hp>YHQWnFzxJ})3O}1*Z0lkdEUW(pe$0*r6#?BA^b3xX!*xA5jRX1 zEiEv1nG_LzYimCHCDTpW7A3OJx?|o*uTZPH$<;IE>X`*)?PdE%O&b* zNpD8g-JneqC$uh|A+@l0HRrp!t14=f(w1_Cmr4sH&E2XHdVy`T*pz8aw=LJ6h)u~( z*L`v4Wq#h%&gygV^4~)rZaHyjU9Gp$`Xy7{id+?@h21j!G~+|+_J==z))X$!t(f8# z6t$^$tMY!4?}9Ja#4f*dY2DS7hA6(8C-D({s?XxHEOV`XxLAJ3y!`m-yq@XYNwqrD zH94kTeyOlx8uzP`EK5I^oxQ&EJ7 zH(#$2NL6}!MM;id%=7ySR}Zd=D@t;c7KurI2%osDQ;9!b@6(1`!pikiKOQ+f!|&R3 z!?{O{EdReuPidbcvuS1S%MS-1$ppx0$aQM;pP6lTOLnEiv(LFQakDHGT28g|%Im(k zFmd9g11J0@K5X>c*PF}!-}hU>#Jgq}GL?Ds-i2=}ET~9XzG~aUZATjy&h;`ceL401 zt5y5t_T34uTfaYZ&GZbN_%BzNUcH~PUFc@4OZ-|eiaiv2WuL>;k8!HfTQ4jWx$8GI zZxNgAsqD&C`?oX)&%EI$dT(3Z_vV$icm7D4&$zchdEN^jj)_-y)IR>E#9iu@GV^Gn z{=}zs8`j#FT1QSk5oB;v>IH*kx!+tP(K|m|*Z=roOX9+F8CM~k?Lh@tyjN+ZfZ_Ce1(8|4$M&LquGTpO9t#FN3lH-Sk<;cilJqQRU8 z#tTJDnynL*R(4E1B6du!cWs+^f^@Zg7(1IswoyCKz35C)WA)V33?f z^TiJ;IqJF|;;*K&9p82QTchyr9n#UUe4!ge*Jo8Q_X#X6vD;=VS$lo^GbMepSXSF?A zli&Jr1H-jL?~k}eO}_U;%xKmf|7R22PZVwHmr+R1sJ_+TxA^&u471L?2dy7mo$mk8 zc-%&ObV{?O>fn&vnipZ5~G)&Qx?R-}m=q zZH29e*o5a-(l5;3IGy>4^e5(b4*MrcbxCVa&0JM+sw=#~&#nDa;V+ZB{nr_P3ICnE zGlK1OqHctucB*heNb!xNJ=->{ujIOaKKaD+n9P-1oT8N1PA%*Ho;j=j!L)yYZ$Fpb zEIqzhV6Jqm)Y`7RzFJog7Rz^CkBE7{RHIP(fpL<{y`>X!F7=f-pW{(X zKB-b_soRi};S`ERridKtMS{B*I`zZ{G9H;4%{_UZqf7^ zZIUxuEhEfcA3AlzqfC8wyKbuBH^JXswTJaKMeRw7IltGYxxIq#{h{^`EB{3Pv;7|@ zrBkahciPua6_wj!bpLbDol!O`Zq~h7MrW1IhMj$WtLUSM=(Ypd;rm`GsNduXJG1S8 z)2!o@KOXsHcRr$9 zOXA16_J@ed-#m9TzE1qp_J=z68;gHSMDL3>o5cRo@+L>9%irCZpZKzm72jwtXql@Z zwU4W>$hL6e<}a$(xR*ZQz2~RJXkRTAmpb_b^M+P2<`>ypJ_q(yPYA!D_eFW<6Mix0 z_1oFK0=2*Co@puk;%(7gWg|XMLHUP91V`nQx*u|JE^9M1?kc9A=KI7bdssVD=qAsU z{W|~V%}fzeyZKNt=-9j|=Pw>M5dBtcE0g}Z;LGoqtJhfD>wi^T9LOx!aMM83jQQO} zuOG`(L=21=n*OLnZT}sq_g#5zUs$Tatu1R;+`WB~Z%g`J!=t^EF7rNJWFOU`{xPbp ztZ6YzucNGdN7$R>8#>Y`O9G#-pYM zQep?CjI^udeU40ioOtDJUQqh3_*XL7KH4u6Z%x-?Lz1!U{F@iwEESm;S+d-1r+;P4nst42Dj&6aF7hIJ@rPwfzbYE#1E{WZYuN zFcx^tP+~V_LFNm63BjxN2Roi8UzTI|EXcqe^pwv=DRJhK?~F3d0`klj+K&tFb5FSS zfXimefsGb6UNV{UAMFfd`(VoSVLEfl0yma0W|p@Mif{C{==HI!XEJ%iJRz2G!ef=Yst~E|y9d+LW za~H?xtkHh6Fh=7|TsViTL|E3xYkOwif3=1g@n=a%he(NPG+O3Fd@B6zXq|Rva;dXI`xBSy_X_m_pU=C$%Ho?6rdRv7 zcZK<@-OK8enQu8PwA$-*V41M;h4l$8JqMVyZ>$yDeBI~l>zw-g^+614enu+p`7}pw zx%0wOV-G*SW4w{y&)qOw@%vv!e$k|wy~(#PopPLi?TYMA%U|9h-Fx>7+4`j&vb*5i zuUnP!J!$+DoRq@Kk)Ed6?tGTGQOBLjp4xE+?w$o%K4rD%GNkJ5}6IGo4#B zW%~~8^r^cxAIZ|rxTmw;b>3mU>`hVIa<-PnuW#JRs(W)^?c`^zyVUvjvi{EXU-dG= zN-lgyphcz9+!Gn)mt=SIF!)YXOKME9D{Qo0-d?oI=O^E+GW*+mBz+EfZu?+;>vG$Y zs?%Z{%Wi)(V|PEVAW;@VFm`Hs`sEGJ1>+rZ`pM60F7Lfj zw6N&m8lm`R%b>|~HnC5!P0}qYl2PfOFkRf;d(zVRdN+^WSI9e@miBpq*5$Q_?erSg zTnyYfVY8D`u}o#K29uUb(p0m9w>@tZ32xK3h-&yh%iQ}Ar@;}kIV{?Dx3B-EI)|sR z^8;`G>7(B#v$gh$mIyANBeZPdi`OL+el&5*hN!llEW5<55);e5X6p4t8seuWEc)u{ zGHaEj1&8&Yb!*tO0={paaL44SPTd_%uRX=$-*{46wg~*MxP42k=gP4g?Ims{FVuou zxXfMi4@(|<6_D+{^sR-xu_r%&{wkATw@mlfr>5@A+4!mP%ep<@mg^<91Zhe-RBJBg z6?_{0c>S{YvsQUe)bi5U_WC`#lVOyd+Iy@cac$0(2Tngt+^p3Cj@8`BI4-oS#Pia! z(?08#2_Lq)AXU%NZO?MkkH?`yF~|5E(arS>FPn-=t*Sgz^rGA1^kl&w=X;I)J;Tn;V=WI%JauzXaI4dq^IfjTOy^xb zqT8`c?^;un<*9kL&upuoc@%aDT`*00aocp#z3PCIX?clJXTRO36xm@P^?3OdpLro- z4j(xT?7cs24w$wk(d4~w)TgVrb}bK+Rr?=x&+_nO_1yheX1?htThx5wQf|5KjP|0< zB1_+~KUpt&NkKZTrMmE*rGY|0bVAoV#kn?WsjEx^&5s%c&&r&zd5+bybEo{um%S|U z-1gOBqlc)8?d|I|V+VM5t$-hmC$zF@Ml*kJjg-^R&wqcd#D%QO>{=RA5yoyzH zgQ$sa$SW@HSy9{thg?M2gV<6l!|K;v*4z?v=d6jtpPg%?w&hHe=VHBZ>)Ey}_s0*H znrwM?SaW;ttgRUdN25P4?w=@~wq@VW?C141(zag5?n}M0SmC>0_4DG_OW!WN?RM{1 zQrxw^u0ZM2vp3s?JGsPTGglbz zT=sAUbNRB+1J0S<4?Qz=s}FAJ%{AMbZpEt{j07`xLVTtb%7=4n+c0i=FOq{r>`XGOsTcS6G^oCvY zidy%yXp{9(-_+)NE6iWcd{c01uBV^ca--?)n6@t(_DMkL=l@$PM;sW^S^U1x%P{4 z=5FP+w>P}_XRGhJSfBrF?EOuZANlp`zE_v+n$Yw-K>Wz|Xr2P^8;ag9qt?1H_jGr@ zDt*azA?wJjN|ES&%j9?8T>gFL3cI66N;@N^Z{HJ&*LbM4!n^cg$hoad{l8nUzW<>9 zLFW9L+G$U($ffXm-oEbMea+SP^-+&&4?NNydX!u+owK$tWnx>591q`p8oq`x z?_%$leK6fS-^p42{f;-c=3U>gKFj~IN38zE?Ac#Sw4L&lwpSfb zZ$7d1mP%6c%$X0i@5~U`P_DH5(%Ccf^$r?wkMY#jA#@2BHYiIdL-{?^saO7+kM*1zwQpu(B9E67WXf0@&u_X zlf)cfhekar$($7sBK$DsX0D9r(y3ER7e5wv5G=6>M6ZIyWK z-R?hYp9}mt(fs`O#YLq{-aUDudHclLcsZqDv;I|edu|@-dbd~pm1X(Sc`xKeoudU; zN1VL6!dh~bPwzc8wS}KN`)bZzD*ZL}7jxCB=^CaPfs!>+y)j{lN@`gh`YQxX*{8Yg z{4}%U{FT!od9yMe<$B+>Or1QX+j3RW)L0Si)w{S@Q+-2ul`Sj7VlrG9-mfmd?7h4E z`pHY*c8R}S;63-{vR&_P{P2^x#Umrye0)1cvvaEi`U##^QT?r#x0Ickq7pc3RV>hKJZoK+53QH4*SVxhmRb+akuzSz{)Hs zD?YuIs}9WFx~AmPlS8{U{VF}QQ-ss;?B;iObj!45R^M2-sc?#Y9qVCHGBEWD7&*3 zw}0%L__D`bGJn&CHk*vB6urdlA2OHiK3nV)#XNNj>#}Lj=55h_y@WG+_iU>fUr)a% z+7_}oWOv)%B@=V6B$=5{UJ*NK#kCA0)5%*Tu1Nik_%*@6N+{~%X#uv$Cv(CaRxf_K zjZ1oK?(*ydF0;Zt?QDci_+R&0?bXkjlc&)3wuGU|hV9J4t1s$aeBgPw*(anU$d`T3 zv&pBHa&>xMEjg>UdO}CaLzC&Ey9!^eYHxfv<@Chh)ED_8qFmv>KDEZpcvXI;(&4N1 zgazi`rZ<0NlSnpQ&@1c2|7OCg_>x`;!}W~MI1T?LFOJj?x~i`?D`|fISMeFrKe>Ib zO5}tloXcIxw%z2|m1C?Qr|P=$v#L#9d2F^ttZuZRy_W{3z-yyPo_U$ieDWOI1YUl9 z(ZG9AD6{&ke*c{tuQkJ}R~rA{u#D9{^0Dx(FD6-&!?$+-zms9iaWlJ7YmsQ&9JlU& zbEaiUTz+M-)T(!y@q8WcR6FL6l2wwYc!haI*-kESZC!;isBnN#Sq4g@k8mJ?Gjk9n%)_Z&UW_pRvZpJMXN# zGF@!jH=kE8xt34yP~Fw&RsVXz632Cl?-(_igP-r57OZMH*D!jQ<8zLm1-Cs{dMIa3 zoA4lX({r!%B|a6(t8B&hNmnVp=u**qaQPhJ*8WgXRvHBSJP?P`^veBxn26L5B7^} z*GyENw7$eceQK=8;q~|Dgg&16!6EkL3yZ-0_q!gRu4B#pk@5YuyzXDiUydbuJM)ik zPQ5$JUZghSR^GPWJ68X`X2jmLa(m`4Z*w{E(8Os+#Jubu&$3v+xUp4URwznv(aCpV zd|4j~*57#cCa&7y+%&1(MNYX(ox*1an6I4udRf5CEA|0GvlQ+ge>e5n=AV~@DkiOM z+xkX9ret2*<#m_eiS$olz8MtVIo-2=*Yq~Nwu%cgWlxohwd57uf1>$KjD1%dTZZ?V zzp~9X&6n6J*nhF5^-Rnw?%}>G9`zzp!s?Zsm{-|?y0E2U3)E`l_9<>{FiK}$mMKtV z{gI>7y+fq4i^F5)%a>wqTW7wMiuKZXnKtXu>2)WT-TWu8YO3PShgTn6uzfdc)xON6 z8FaGiQFkat!MP(M50mB@92x- zeL0=;A}(xslNl7?`la7>e@D{87sn%WCl{86EtEaDn9(bc?WNeZ8$4-A*)JuZ?De(2 zcRKg##(cAzL7yhhyFX>()O^v8Th`R5I_h$E{fy7^;y(I$t4(6n&7fS%G`)lx|peV7m9xewVpvTBieslKs&;6j;L3`*wQvmHa&D^oThrobUD4MDIq|%;c?G1+TUH1oiHb?2Yg$ zY7{*?hc}V0nrjM=^`Y(^IorL~TkKodJWnLSSL2nR!&T*l9*qCBqIN!CT>ZW7+Fr;1 zc^kQR|GsayVdMVS*Hf;pS!z44!tR#Ot$UyUR?oAWB6jADp{Z4;Nqt6LcJs14t<%}1 z{?jjRJ}PHm@y+(X;NL500=)LNb9kQz-=em{I%1C2_`-#zqaS7q`Y1pb>-P~ z*{8wjQ&;;}c*Wk-Ji_{M1Iu+E8wrKZ=W3G!IwwUfy}aknk)7(s+9FvK?{7+#+hkw( zt@uW)-@Bl2XN{+xFgj_o~mVV6U(I^V2@v`?l`u_`Wq-8NPhe zL+)7bem3*_qWya6@fzhi5;qdPmOlz)ynbO$*!liBAFg#6%AYkjA8qk?+Puoj$5LDF z8s)zFr&<25p*W*0taA6So@MDbR4$gJ@UO6sU9fstu-VT?{a$gCYg^1xTM{d{J9qDx zvHI>Vo4kpwFSJURpV}9EMZMnjD}Q8k$7e3zu4NxjWvohx+W+s;dMB>1hx63`WLf)g zy>^}^6u0QF!|Az6$>ooZ%{zZP%lh-|t(SH5z0!50lmoI&Yi-|5Ed3^+``NPd{LRyz zr{@aK74>^&vNByPFuc9=YbxJE!3VRROzAxAeRE0o$6UX=3IX08^GrAH-Fl$*<~tGg z9Buvkwn|Z-e;&W+EnHq6`TxMewU@r#>Wz2Jo*(bwW4~eE)==y1Jh#`{eHY#@w`Qi= z8vdE`@2&_qU2EGmVK+lm6nBp9U2{kA*S_2I-U&%$=x*dK(LTN9w9b^m675vpc_Lrf z76+yqMeDd#Pdcksmw2Xk+n$2SEP3-6xXli|@IisS%BtZwcS-r1mWJp{S(jz=b{~GY zO|o=XNjq<|+3Jf)A2q{vFN(;uT72#81f4L}ovM~SLdDu*-G0lj25qs+n|(HmHMi^Z z)J0paPrUblS^1u9hk?hE?_u*RGA7Q6@qL!-6R~K@lXs;rTCXitxxTYnr2oyAyQ$S7 z?_0T&RpymlWv*TOF8tS$aN#Zb#b-{q}kgf0f=#`=+%kE%kG) z^6o1(`q!e{cc1m$anG|PVTaORg;u&0d?kHaj;fIWIAL>XPlMKXd-o z%-%6`b+Wp-qN(Z%J;$w&-t*o+&LF;Y#*VwL?~E=8bWKVzobBTh=o1zlG>P%Kn=f0M&n4-OH5XLo z+G(C-x0+HUyI5`V`4ey4lrO(7=?R>bVkFtSAorC5-xQ_!hB-@)Wf?s&onN=HZu-2J zt3AI?Q!HU!TJdNl-|MGm6PHfk6rac?#`U^(LQeUcZ_^hSJpHmmA!YTMTC1NqH*Xs8 ztaub!HnryL#C1Zd0!(262^%?HYF>G_bJCHe>Hc^ADu3O*aMxSOmiO~w9F9zT?x(w< zGrfEzt7qY%)ye7GmWgE;2XPmamW5UxS(zSweX2Y}f%#9}X3mx^HLRxw}^?baG5@&G)^2Gj4k9yK&@b3H$(IZT@n7qxA*>a=*CxgHPLo^>|5eeay70;@cV_%d>Kv`GC7)ng5Pyw~sdnHN>@znm6k zZPvGc>U~JL{J^Z|Gu*{w>||sYzuf!e{9$jG+mROyQ#Ri!tGjV{sZ8%0gIUZ+)Vzfj zt}hb1a=_|t@S~##TlZ&9{}KJbaz*R>oIXFxni)DBeUVD(x{N}s0)x62; zd-G<+rSdIaxsz*y%Vw_Bsa})mQ&zdz>++j6f2;Xkl{cSM82+5f7CkvWe}j5ueY3x8 zyy=CqBBAxCU!TjIGB^GJx76Ju`8g&I(X)zYm-(DCc`Wmo{g}OP{j-SZr?WZ3XQe&c zUG%r?`-C=OsY`A1mOWGYtn`_y_>9h4gM5Se#>Sg(9*(*CW$K2@dyl;Pn6bdZd;aUh zwl?Rf3kANUhh2_7)8-TRFE2m3u58tdoV%JIQ}~M2kIJ7E{qwcXP3Bhf zq56Qm-C?h;D%pGg5TCWCETq(Y4qyA-ms~9~))+qC>6z0W(IMJ*!>sIZpI&fYj^Vo# z8k53p%758r3VxSmmz($7OW*bRsT&Uqd2bsh7QFjmxOhJI(z9>wSH4{%XIypT?6N6m z7X0fwW3XkYdevUZc{cv+=Uv_>t-qtT@KnBP=4-Dy`eri^8_Z8HA#r$3MIpp($9z6(IvX3v&vj3*R$^NVq*X4ZY7jv-~ZIbv^ zQu%74hy8nM-=Igi_R=IPx?0vW0)mHrNTp?R=+b=wk zyEgydQ!6i55x(c@Yfh5`q{5PumuAX{T z(`~J0r2VwM#N?$bk998Ny4-zCch}lqx^9avTipJkoBFIraT@2emttv(P7(Q8%k182 zw|8!k2s`9y(SLj1(IkiK7oHhr+>y)s5YeG}dCtVH#V1UjR9sYd?0=}%uUx#pt-{n* zYE|aES^M3$YEC})D7WZQdfK*yO^>(LmVIj7Jo)u}xoGZLS}$a#&s|mC*6Gcm8Sy#y z5bM4-jNEA(5A|j~kFyo}Tlsz*8Mv|0X#r5FBCMRG1+$+>{^{-CXlwZ&Ge3%`lZXZ z^?%#<_1(n&ZwlvkEizlSIK^^8`tifcdma>C__xMK>+)Cgo|MN+ICo8CTg${1m@)0` zhLV$YhOU?ITvOY*bNSx2)n3*9`L`cDv75JW+eb}j+w#-p)_Rw^^H}E`Nh;b|W!bv= z+#I>nNvTVnL|d0veUTDqTCVIpDPTuTV(I&{d0~y?;mkuuRh`#o zE>FoiH1p=q@C`Z(yDjg&_WK&3_Vh{2oGFW|ewQu}aXkDsHz@H@Y~>-x874d9Zam`O z{zhtl6^p{Z30dMd*WQ-?yXS5D3(F+?vI#Rh-%8eAUGwns>%I&3j$LDx+xI`!=)l)M zX^&i^6I6H?9ba6pwU76<`()M4)_rw>Zzg%}unm2{d7X9kp_gT{8&%AG^QOu8&71LO z%E!#hr`2-1RyJx*+M_*Fz5AxA$H!;OGkwowe0xWu5G)zaXX zM&SIaysUdx@wLBJey{i}o*aLE_vG>kZy&UGyYRn=iq$!N^5ABPce^X3Y@^lql3J7g z+jigE`K#~Px`@=dZr2`7Tl6AlWx`b5l`OH7-bfmI@Nr#B+35Uy>xq2TbBk8AJMqd^ zzTgX48!>T;QE}AvgDck-|1P<;f4;3{{iPj$&N1hne_=hHe|CJe^r?XMjQnr@Qx=w- zX#4!IsM&Foxe?3TyF5qBBBxKAbM1t{CUY(kiKtuoW+mJBFHDhelh%7#Up!+ri&W}X z?k%TZP20NT;wiUT)Ba84?UpJvEBPjF{^t4n>jgb&JL)SAe{vJ;?Ms>a@xP44^wd(z zRCm+6J9ajzdoGSLINHu0nYdpoL0pDGF_ypOwzpirYy6ythOg%K?Mawt_o7U`K=7z& z+0@-yF?ULrx3VZ6p1t$(ru#*Qgfl0qI&pX>R z>aCAb!bcCO-H!P@uMSn}eonU9EP6h7&vglpoqnH(4vm?#2o6>I1@bZZgF%dZT zp67&S_x@)=^5qt57+0SXy_q<7^Us5^m*d{`lay;O;DZrI@LFT>FaYjw0wd+supIrV#-emvj z`Yc(qy>UrB!I9TqltlHs*m7%u+qP^IqsKNHFW*jDa$UK6$20~0Bkvbv^Riti{ut~K z)F$R>@gVs7k10np=J7f%aQxECQ2TY>4z-hRm#_cR+pao=b+6Lbz#B_`r6+bK%jPAn zWBPclt9$)vkBSY($qtJ%xVN_Y>{X7x>ONzaTPCYjZdBAl9lxwh)=d?+-Ugp^tN)Xj zy7HGM z&uIHj(ONR&$kG3Qog-r;UAK91L?P(in}k) z^2$+j-|H!8y?e%7=h~K6Lfd!52AXfs);+W8j`(rE>O;RD+)IzYDI#$v@6qGg-fktc zFSfN$`yyeqCgpeA+S14`GjF$ViQKg79cNVO_q%)N)hu~eaOhfH`<1Mu zBE!58zoL7#Z=QI`u-O5lvkNM z!|vCI#s2qx7U-ARs#~@Fo~Up3rR7_Dyn$BlROu<}xb#1{)?9g=ox!u@sa4Tst0lf) zbiZZZ-f?Tmy*Ilow}w=A7jDQdcop&1;;}^Pp6r-cdiT@6t-lk$v}Rh}sXDE{OZ1-= zvsLs>dYp0aw#M9_XHO|S-SY7GmK8}GW?kGZ#vXOElRe5nJzegCu&eyF>94iF#MG;M zo?2YFP2s?Oxy-#4OH}L2m-X)F)!FgMV|K{;6ulTODisY zxbo>s*O&88o437xBH#1BM4LCW?PLAcZ#owWMclugSyz^xW^Zi0d)k&sIsK`Bb52gu zIX&$<+n!_gEy>(EHxC|TbK9+)`R4D5-J7FW>cS@riTR_ zii$f`&K>pS`!s1n`%VYFCm!<-a|?Ke$6o$DF;!tFcT!q*)vnd6R6Xj&Lzs8Yyf4}`rTNF|Ki(W?&!qTGT>L5Y zQ{7F?=*Hrdw9U_bx>?^kd^l5S{M7kd5#zKIddU~>Z`raY_hD@G{F#5|fAL@c``o?X z>d$P;O1~{Ue!Pg^XV#t#MaI|9?R-D?&kg78`*$Dv_*MSjhuimUzkYsyDdUma3H3OJ zmnHpk2-WRjyeqH!W^wH5r zx7u>Qy8X5LSoZN)TYFP{cJ;Z3!mG1Zt+q1X_IJ;!Mbqw|`s^jH9{OIStjl`o+<9~F z&E0qRP+sBoEw?w{-hI1#$IF7vKHtwuiRonDP|W>)WcDN7Yc;DA>krSpG<{F`mOEF! z?L5c6)O|0%iT#bD6)&xQr*3|EXswm{Cg+U2FJ9-Q)@Dw-eA#Yyn;L&<@!{2nY9DGJ z?r*P`DLeUa=fjd8=84bmd{}k_{rcG=>({qzi?eskYAY~_xGQUSF8*K3?nK2aHeuqHJ10a(Ty~8% zbv#xbdc9MxF7*08y?x#fE!H(=@D}~e?>cz(k<+Bz<-)d+^KSO|&40EtJ?iG9Q*j=p z(FbRz7d%Mw6YqZ(dNbH@OXbBmKPU8IjzYmWmZ~p3jf> ze)atKojz^TXJK^So}KnLT^f zZN=}A*Nl_(h5xg~SO)6MSvJY-wDdb<9fod5f6HY*n#m~Hy~?YcE@ zR_y(FbN9BH`hQOM$3Hr^{`O?UYxC^)@7lQfd)ry*W6{sH#g)7*_&RNFbk(!#XZgH% zy~Ez$-uYNtzP2YS>hP8MKU+n=J3QO_-&5(La=x$03_HPC-UI&(QW)drAF=dUs zueqF6k;;u*M$h-H*ASQZ{(0kVi}IvT!P|ZB>|D9F?c392gRHoN_50t>RQKeKm2Tvi zc)e-rmxe>p7g@7b>^41N%5l(3UwqA%0}cERMU8Dy#``>dk)^~N#qVa>$~G-RBPI%o6de}Xw)_e0GS>blOtrdrt{+++Cwco+%=tQYm&zp2q-gPB=EB3DC zcTusLRomFX@i26|L;jzSEAPuU`S;a2_5SbaDt=ZvS!~C_%dZ0k7v%qpkjdS&kaJ}I7GQ!T#nM9=4@ z2{+!{4c#Vw)vK#|h2x2$eTFlCY`@%FygK^-hbJkgZkiN6E~rZA*?;t0L2OHi40G>_ zKL=MRweM&>$0}4`_~6rlw~X!$x0YL!$88TXzkM!mas2$-7v7ds9$m0HjAyl|-=mUB zm;U$wQK6ESJEcx*)6H1&d*8J>t=>Odr*yqQ*stqWW;+(W^F8EtUaHdhLDJWv%I;mq zWyRaB7G5{a^RBq`bxKXh-|pA88;=TSDC+J@tM9(ErmSV!WI0p)nwTB83#Wfid?)x% z@VxUqPj-i$L%&^4Z`Bm8cUouIWf%Nt?=Q$_zb|(vQ*qDBGwWinKKS;qK8s=Zs`Gn; z&+8~PeBYnE_}<&_bBo1WUvB+);Lg9(e7{ZBAFD5ZC3@)K)A?4FGfGa)s;YSZ;(e@NWqkgHFNZruK^Cusg5HFDwa@0;EPcg_CZ^<+|IWKf)fe#f3m z5qI|0A3Xk3>U8;sSgv;uylb_VfBUuW^EK~H?}XNOdp+>zbB&$8=-O4<~(B+beAUNbjEA z&hD!9m6NAi{p0wx`eB}(%Jo8{r;iTq^6<`>{b$FMj$exu0@g0w9KFzddw7U;?bWQ! z%WvL(6a6gqS@f&@>JKslqnpG-^TM~YZ$EDoYMgOSs=W8-G3M(n>t>ctelOg3`^xR6 z?QQR-$$hK|uC)3P^5XlY_fGX6eKp>&4(qI4B_4v;!T zijZfoPVS28+!vI3R=79oo?rg_>xG@=T<5c5DpokRo%^(!?|Rnn1C>`=rgHu|YIZ@( zHY0BB981|SKj%l`!E5()3angq(AV*()6ZS9R}HOOH-2o8cS|wtIkCL6{8j0rnN@SO zrTZ7IjF*4g>G9Q8(B9{edzvfLn=ZO+Uc>g9*`8P=VzzF+<4 z(A&Qu=XDMI{2S-}y`vjAziLi>;DgY0-fI_muea~&Vki(lbFlTs^F`v!8yfNjt*3B^ zPpsPc^-tBp@HfF_Ma+L1HaPEb@hmX3W79mbb&hI`cHN}io;yNo+N}>?Z);V{YrLT4 z*0P>!|ARYMs`@oI@2dRZpk%w?HqSzf{Z3OV94ARx%BgY8TYNv`NoiHfA=Q_5kFvjd z|0(pdD?``+pJ6uVyuVl92%M?Ry+f_>neny=7A=*E zs&^8i{8K0X-8yIQ?VYhJf*Fsk^r=2>7iS+PE!cYM*5{Wc|6N3Ne`SZP>iM9!BTLI$ zx#DEdIn(*`xSrfOJ?+fwtB34o?9JJ9UU2@8u&NsK@Au0sSgyIo+CeqT(f^`TOa|xu55V?mESI@wdy*=~Fmc-m7dD|M&RYBc}Qj zx%G*a_qRXW7I$x3&dS3-ug|af`dC{(cEiF?URyS*pICeDXJ2_Izg$6Ji^DH}{^$?Y zLCcS+srSpxo+D%Zw8Feti&@~f{ONxt&j0tG{n~f-$iJ!OPHH#bUk$7gDrao2e6;?D zspcQ4Kcb@Q5^r=?2Rp9P{&yll{NwIt1~+`?m2WkSQ1E?M%OiGDy@uZ?qR6g&isl=| z=6_{u)h8$XH|F`alW+IEqz&`meAyrSNAl|Hr@B0C3qt=1H&;Fi3pc*uXmv%8&HD7K z5P|i#PA7<8xi0xet@#y4Gpi7f7LVA;U8mUCy!Yt+Z#gKpX~(6M4dO!c^$l)3zp=1x z{=vFSai06R&soYIn7#dzZHqPo==`|d8I_^Bhw^mt-69-zuqRG@d7>>>_U4AQ%F^3P zQx2coWVNj?=f<^(ytl13B}TiO<-A6b2TKaJF5UF6YJV{sm>N5VfA814?j4y>65{>y^;^3mZUzO6C0MECwZsgt9|QA}&Y1-6k?pjHO$0<`JQaWC!u+5jM6j{e;)bv5EV4CI*oiK0Z><606276xzr*ZSE?}Q0#x*647hOW0JaMdnW zUMg{TsdMLDqfUuTez%6-DoG{}dt0}!?P`#&*M;?1#Tak`~UNvfSiT!lr#a zN_nROd3K&sGX2$dt^e`zv-Nx{Gd&j0(H8y~{lTvFeB}|_Lo1}$<*A<|Q4^M2@VWoU%;Mb!wCIW_>+!NxCr;M=*1FGIX`fIZ z&xMbzc| zecQ@pN@CB~`z~u$H=NVxGrlbqeB;tP&z+^ST~A+n@b{{ZvFDSvTg%pN*O5NisU+`N zqZPTnblU!t^RDifSZvrKQKa$mhskY|6LA{)9%k+P%K~0J6u7mm&8)!mlv#IK`a-j| z?x^K^Za<#ew{+X1j8jJ1+CPFciz5|7fzcp?J2-RjZZvQ4(S9;WBZD755&o}?AyPI;7zi{oF z_UqKsg6j(R?zdQ-cu=w7_{AopnAr!`dHdgYus*+`aK6i%>MpB)`+wC$>mNQNEOlo8 z)IU>tOSR9iOYLO2oxOzfWo}co`~^9u)4Co#x6;Ix+8)}lb@JJOe1^3CuLWxxNNWEIh%U3tiT+f}CM|o#dFr37Z`FOC z&E9;boJUz~@r&w}Iopl1u5|Yp&bN)2^EGGIHFl3>3DG|niJROwVb+zb-duMosV=$W z=HH!9LM7gYB)LgWo3wp~r~bJ&-M2m|R3*%Q?YL~F$l)42Ueo0D%e+s0`(wN|N$qJ9 z|I&Lw$2;8ES8=Mn)%hIy-hH=hu6^jU^T*R)97?$8^Yg^UbWT}IDc6&iDo-Ez`0b(9 z^%=U$PSs>I>dF2soU$d-EIRd+(zc~voDUeS~uPGztt{# z#$_ls_co6h=dHaPAD@!QGA*6PXOw<&cgxIg`n~h)&;5BNaV()SNbYFuj&IyQ-WXDf_1;8YNVzxd(-v0SEw=4> zX@z04TtAf5+%nE&yFX(^W8IGF&v>&!O*StoJY;{Yoc)?V(^|eA(JvHNtg|Y3w^L~C zkv{cX-KPI{ihPn_zERp+&o%YX?!L`$Hg;4e)LfCcl-%}hg_%K8#_UTgx3Ie2npKi! zGWVjc@!U&p!W+wS*XF&H4_eu*botliP8r>kS$j1zwp$3jjA7Znt}M4yX<@EmyZf3o z+m4=)nrW!KPi^j|9u?O~lGis`nKTs@dvr@j7F%3hw)WhNDyMEo-McN*PnI0J;8x^m zo0_JpYqlu;%M4cDP17V#XI<-|19n<74`7AJ+W%$VL%!#ziGuJFz{%988kK&Dj zXH!<3nUeN2sBN10TEUDwhfFKK=wOS5%)A-vnT=bGwDXtfTyD5!$WS!%V;QgOBspUv ztCI7lBz8@03E$$V!ybDkDCy=GfeoEHJWqu<8777+%=cjyG!{Lds{1DEnv}o9(@6_X zoea`)Q_Ebo!A)&1`-N8r>UG##f=)!{9OY3+>b||?(50T$3ynRM4c9r|-?ZQf%h8D@!WwbiH+`pF z>~`#SbUvNhyF=GEIls?nc6Z>Z?9!w6B zzqZ~mxZ&1@q}**67F6Cj)2H^O(5q>ipXARsE9|(|I8L?LkTEl|Yety-*(H-Rk6eq} za8Zy! z8h@=Mo3hD)dl!~1cy+Drt|v$iz)M%}*X3gw zZEgPEZxipCzBc>5{&HCUjZJxXmzBg3s3O(?DBg*k54*v4Gibd>*@ zjstJbvP6+-r7J9>6S#WzYjhqaNf>UO+t(^JFzJotoz z-kiME)@#<&`qJixPo6@7>ViQ{wYS zNAKP(cprIb@mHN_sYt1hp32iqcP(11JpcT5b?&bIrIVi~AG&ZY?TgJ+RvyO$E`wZ8 z{zDvhdF-@#`S$hlUFwlEE?paceB;Dc$?VGOzaPAl*f^=PZ^_h4lil2ns+aoxb+Oj{ zV=eupmt~5S{Jj>tQUlART#mL&GuSROsI)ffEoqp;ulD)whd*p>Os(JlO6Xjg(7~o# zcwAhDUG}`iyw=yA$AkNhpDK7MAt5razI#V+=9Zq!8v^r-`-(RfFD`EWT(iS{!L~ll zq&cn1%6Ayw*|8rK=+`@xVqmk_g5ziW?$T0mELTgIDi|n0534>y|3(wDH9j<r=D*SV3m)|4u3!{ECk&m^3%~>tzd__@71R zzT3E4DV`MaEI1|G!RN8sc9MvTyGlKa7x$8u63v%Fo+VCN*XF5+?{-L<&f?{5R9(Jx zrmbDN)&^GnlXs{6nZWG$R#kH1NgD?XAxFnqSzDJX@IHTjKw+z&`4iKb-iJ1-RTj-T zelVf%yVtuz_I%|JH#(G-o_o)y&!eZiX0`E$9l>QA97XThH%?l27E9!qAtDFgy&p#C z_kM&Fr6!doXQY<+2J1TI=cI7y59kFTp!+01cXk*X80s0BS(?Tw7@H|Xn;9GH85SW185-(Y7@C2uIWbT$wJ_E*u{1MOFf%vQGqx}@k5w==RERdUFw?U%Gyz=~VFtU< z1EDik!OTJ-+SJ%c&)m?&Lc!F+P|wuFSi#iLP|w`Z0%V7&Ipk&s0|iq9Lp@7VV?zZ~ zOCvpVb7K<)BTEZCW6%vDAXge$n&?>?8CohBD41Fr=ouRuDVUm>>RB3sO|gL9L1JQL zu4iOoW~g9dY@lamW@4ydY-y%v0=k|Aq|3s{z*xaR!Q8@F&&<*gGYg@uuY zf~AFlo{5=-1&SX*?uPo&Cp9moYSG6OSovVveqX0*=XBM5O{^3K@HK6_f-51zax7$k=u;p-kBHw8ZAD9?z&`w=+K9 zRuq1kHN!{9x94`sMV)<-YTbsH{?1*s?}5>*@3s4$|Nd-y{>#^rxzVMoUR}AlI@Ez- z!vQXVM2A}E)W6G%kL59S{E}vP7aL%=|I_h(#$gORclZ))?e_0k8fUY5%jF23vqe|0 zg-*F$e)AjCks3aQyAihafv2yCf7|Q)_$GteigoLB`0MX35q#^t8azO(?g(K6!_KyOKUCg_oW`xSrV}{L(CAt!LA( zYrmX-g?UD1Z6&Y4Jf=evI$Gg6MDa}`b?<~s*?OJz2A-w&?W{raDYZ|96#yas`{+}?_1*)KL!-x_~G;-bg?#x-si zcD)yvqq0Lx?Na4+4)*5dk={0cd_S@K3(NCv`R0+SY~p2j#f~v}>F>lPAB^XQHLc(Ts95>U;REdXY z(&XMZ?QW9yPBm!$AQ{BQGc{Qu^p^j(iE@9U)-+dhW<8O-;BVhvDdYg{xkO$S6c8V!x@KjwgiXy*v;ZPd?oV1vYpBqH7pnR9?t2ycVDq$-umtR z*FH$?`tGdP*nTWey~5FoIq%Wgv*+?}JpEkqOnS@3l9%jczBJoqE%9 z%zv_c$z!YJKl*KAdFds39~CWpHcTkl8nsh$na~*(iGxqhrJPQE{xp5x{NI{4b+^u+ zX#X|o-;tbI=QeE0_D`!ee7^O}zmxhI=fqil?~pn1ZdcSN|DPQBy9<9BNWaOs`SYZG zg!U=7YTKOTva>mxR;O-CPFJ3vT9>?~^!djp59(ajK51^eBYz^!`nH~B=}GRQ&hSmi zcN~8376~0O+Iz~J^}1&1!P2DstlvT@0%thZ33oN^+{7o;Z)x0H;hY_KCt7(wk8rEv z2Z3(WO>9#Ark6RkABtJ`y3@tm(ES!mc+o$U-q>(JUb=tWE1V3+ctAcdfT3Pzv7e^ zSWY=9G)2**zEf~_SH`*co``vlMvF|RD$Q_uziZEgjFKH;OT+H^{7!RPb|}MM?T4Jv z)##tIl9$?ix_;IqUG3H{d_NNU+outHhEvv)T>+a)Sk|}d?nn)K%i*qyF1%v z{3%+xy*d5ny({z8=Po%Lps)sICRG=FR>nw1n`ecQTX+GmGqk2iBuTn(3Qy&V4U z>~_6x;)_lHY${(~U$%A1?;AUxw4ad6(C@O>52~r}zomQS-NJ9SS^g93%{T8@?6cIS zvh6vi_0PY@ZQ_H@Ju(yhbk@G~XM*w3E6ce5epK2oll&t&WVwCZn|AT-I`==#DL0z( zx#j$WM!(1na}GYTJ1@P(UH$FR+~;=!V)*ABGrbVc;ZXll;%VoX2a<&>KYu#R$d=+> zJ;l($P;}cW{Y(F*7EBf7zRa;sWryD{fp>y-O|vBg_D#K2CH6XUVoHcFUl@O2>XlxH z#$wlLCo=W^Tv(uSvTN1MGh*2@bYr%=yI3t+G}mzB-AC(Q+*7_cb;65xGp}e{E-jzG zZ{^=#%XuYc%rn#|E(rTJVPQDm?yMgi=T8J$?!33>Pt(O@uH${u_KA;{EIJvy%uicy zrrqOLGxtRaV(Bk%gd;UG-?YH@Lwa_B-@Lf*^U?kP-Y%UUU-@U`N&QFmb+UG*c24!|SFxS@<@=KI$@~7q{fPS;%fS}e z$|s%i-uO`7t=|f7YmML5w*Q#XGRb$A%GpWpUTs`-ynjpilG87XKJL97>%8k*-JAR0 z_T9R1+pqd~*<0K8Z})QE@ZLYYKiNRy50Ao&29|~f*#*h&$G7#Z_4zC7Z53rT%UZ1b zN$s~2=`)Y>EVd{+z0ve|y>;2?kMe!>^=qE)kbh8d+wk|ty5q6Ot7Y~+d-3n_tBua? z*JOS^JfYQ~EX^q<8(FdBi^r{rjPvjPe|dL)&G(m0{@ypNK55_i5mmuDUyiAdUv2%~ z7s@tTHkvvAx8L&>|6bC2N4UwmXN^GKvlC(!Jb7)=1p;dtCx1}i!5nsQ{zJ_QnR5=( zSp~vt+TK58-yvJqHZ`M7`Gf45_Ol=Ke@uMm-haq=gZQ@tn+>G39_Y!V7a2{{)4<99Ci)-3#Ilo=06PE!IqUEccwimfG&vmaVTu)Y1jq0|2QrC2fZoVMT(N^(c~FE)koI7@eFJXEjX zus?cOV{y~N6^ms)^YN{-7z-mi)m` z=IBEUCR+<3_e8#&Lor{Hf=2p`5)YG8q8Y-H^{~{#1<&;;aht_k*DGJ!~G9=E12%Jhd(qmIrzRn?7(Ep zr3GhRH2ty3&pWqb;yl5_>!r$`UQoOl*=&6I*LQERMDN`#<;~` z+uC@aPt(qS^q&**=xndI$Bc8o724N74KMnm6@NPPa7A0_y0t6UZ@YVK-QhQz_Fc1l zXZh6F`%jXkZ`y-7N%6uD!II z+py_JgqTT~gzYw&&osGpqv6>LGpx9++K;Gwig_iRA$jlU7Kz874!eYR?!IJu*VpXH z%!othj-{Oa8f|ei)*{o5^ZK@jlXGY7E+~?IU7Fs$>%djBTl^`0az%6AbT}@&XVYbq zn_HO|9v2-MU0GdlG-tba`Sx60ot)L*r)|vj_Go!~P(1wT&(C?Y-(5ews!${PzHOD? z?CPSE8!Ym8oX$-VYVOWCto8Ggh?!e%>ElzL(Y8B^KJ`qRYc(}Od-;@{#V%8W&9Z)7 zn`aWX>9T9!jPG2rd*7TrdT)V%rdZD6KS$dhl~pl>eF*F<|M0!5jOE_)(r=0(Teuh6 z&Ha9Hx%^w9IYx=b%W~y5{=9qk(N$LF$LRt-Y8py2TnuIyJD&tXTd&asGni4+0N7^({BEz4ImJvbOJgGuhmrSl%6b7`?j#;@?Nc z@Bg>%tMs+~yW$Gh%5A7RnsqU_zdW}8>-Due@(!2&d)*hSeqHzYht!X!V@|(|QEQEt zu75lCde^+87LGF%{3fWK2#WNs?cVRobH4Yy@9Dk9-yhz2UL@+_xll>#(7MV^hurrF z_vXzuzxc}6pgMMzdECcu6ZYI?eJ}dp?Xsg87iT$lUnuF1-JBVAc1`B2MV_-3G=|++ z$JWjD_tufs-rJP!8Qzr-l<9vt?yL84{Y$&O7v}qMl-!r% zUEDLB@$T~Pu3UkOHxwkD4D0<>oYr7P8+4O4a$`Ivy?ZFvyr>_w6YTe$F{6aW?O5DP( zlTPo;gL%KsHE^)J8NDU3sVh1M;eUW*x8_s*UF ze&qOf8SkIH4 zpDwbl=l{N%e(Ub7=W6D*b(8PU`7-~x{rUQN|L@CxGiF?KAo3AcrdY=UCw>1de>UuJ ztcqY&PYD!X!LPiRX-R3xoI_iJAt$X z;_M?L*`1*m))-zB$?gsn@>$fBCiyAE(0@^l*45Kl)dl~{V#B#OEqC6{dV1pDTQP5r zk4I+uv|QExY1FhvcZx4-lY+NaYEoo_pt9Qy<+derFa6Bqo1e7O&tBB<$*L_9bFJPw zw_W%kaQ{%y{QL(ELGsgn3At1mFi*S{{F&+5&c?+~j?D(^k~McTf8~>z@O|#d35Trg z7FT5?eQ5bI<&T}q!mFF@p6@r~GeP&e5$q1j`@D*)*tMIP*z< zO>1w-^691>jE!sEUzJ4UKFVm%+;aAbfM=PCV(t8W{Vk8Zzuz>jvAbgbX|B?Zx1Tqj zYh-@G?N$3+PDtTLz}u&OB8y}MC0|DRsUBo4lNGwORQkE-{%wz6_s3Uk&pdg!vGz;O zLDd_t*Vw#wC``C-exGIE-tT{xe~A0CP&__i|HXBT6V@&BI>I7s)7<(;hd1=>O4aPh zsEyawY|WD1YI}?EZA;q4loL~nLscUM9SoBjRWt(@Xw5qH<@}bpWnB{bU*}pat$z^x zSg&g4g%IzOzB5sFa2&@&R2eBTKT+&329FYF6GGFc{bBFEb*4eWWlZP zRvfcR-kkI~d8t+g8>8d))~8lqQ*6ESuU>y8{Z;mt+~0;eC-c{<%%?onhInwA1?O?)tj_;d|N7|M}A&YJBfld2OR}IeQuFyCc;Fr}#gu_+I7D zCG+d)>jNh@zOcF!XkVuFPg4Ez=FXo6lfUHrA68hM9AaY93$1ItfA9G&K2;95PR&K}c z?E=?7w%*D6`C&ezZ-w%O@Z5iwpR?UhU-^%xJul#?WF7a^f~F3$lT4KlFWzI~-WGL1 z>E+I#bz0U>Q!KsDO;4Th_)Emml+E1}7YjbUaZznS@yv=f34)D&e;jwcGk1 zI4^yySeagt@TgaQY1tJob|%wPXQH!~PgD$;s=L>{Y@O3KbVN#c^q;DEMzvF#)07XT4{{dl|1#;zzxRKae)pfFAlb`P zr|s%tc|o@7V4VNzR|Qrq8AXpTvBb=3@%7wsr7Njhlsm;|;-;lco=39Qx(c%_@7tBs zDED|#&=r^TMH4Lcg*vM4+@U`0%K;(rCIR8=i%%qR>0kOHVe5MS8uu5Wy;1oW!(YAr z;#;-*-?X{_`<3m@au2nPW*({%>r8YNEEKyaw6Lv3R%md*tcJ?$7iOSO#0IG#5iHGg5~MQ7k4c!5ngm`_Vw47ENW$n8@lvJ8o@bGvsd;AGXI-CVKC67j)N?Tkj&@65Eb+*H z65?h2-Sw2xQhmjvHy?>8oa{-OW_e$S!PVvr*Hh{4%ac+yx2;(^Wl>N5*7aNSx4hr* zexv*bZ3oNaPMoUUix%$8;CZU)rW279aU_`gu2{EB)Woxo0-QBlSN}H9{1~Eny-bt4 zWcThMj_VeeP5<+$3b!V)oK{^Dvw?q6g8Dw|^-u0u&pR-2^}3ms|2OWs-1yi+gV#h{3O>BZ$e;lg4E$Vudc~p7C_Xz(8(d;#=5;kA*%FNm%J9UxWtANrTtuDW< z5o?2HhkBYGwFr`N+Pe6tqQ#6=EDlrVxIPS;cv2{@qd#WZmF?%Z2mGJVc_Dn)hQ>-B z>&ur<&DqcWk@>;XF7u0P z_uRjs8%lrwQm&~!DtkG8X8T8<3C_&%@-O0l@p>BD8U+6d+QrCrz~IKq$Vo|OmtAUn zZND-ZI_PA)b z5j*d>MHx=#o}VdNbup07Xv>onBWo_D=Ew!p?{rS=Tj#JoW$pf#6K~y?d-FMEzC*&T z39{+8-)qhP>+|`;;@OXXeGRL<_oJHa;hzJ`-~X8uXk1$r8G3Qctph8~qik+2?VCHZ zIC+-+&7zuu!}Gh-*WM`f2>WFfT*bm?yDNCtt}PquzAgC`_33`tDUL{yQYD4!lUG>^ z9COW>R(Q+e>Jt;y*_(WhE(^QNd3n7Wug}D^?vRvjgV|@yu3nOiKX*D-XOqee%gd|o zau{3;EGxXR$@1{}er}di=I?El?nGalboqNuKwV|Taeh_z@8wsF<}Y;kwDJ$%RhBY| zSDsgL47Z8Q&e#>U<@TD#e782f-4^E&^{_g*%43$# zE~WJm8wze*S$UXa&-O?2pOjm86{y9|+57(c_ZMGoH{IvzEUDsM_c5i~HKkrfWcJw~ z@9Q$Sp01gC$trX55)Xqs*-5t*5~@E$_xHw%t=Qt`8LD%lW%2vRix%#A`TD_5)fDb8 zvNKbcKY8eP=1y7K#|yX3CTrBm@7{gADW5lYN${7B1{r_*b-gv4Vsh^=3deaVwppF& z@-@AZ=<%evRa{f~aHfXx?q!y*t-jBE`}g^~%BLF)-d}s&{@uRX&1?4MPe=FlSt-{& z6=YvyxBHxN^PHs$U6Zy6FFd;I(ynt_U1!V0Cc7?V*%09vSiH=F^>>B((JeJgyAIvu za}8Ycc(a0h%eh1K{4x!~Ater5)}9cFRGFo-DC^3?xjdUcJhHae`K?s>VZ&w?R!=Mc zfRY&#CCWdkZ(;ST{T}>)eM*z>cD7l!TJr24FF0_}!GNRjdtrimLhbLgFIzWP*q9mw zOF2DHj4bU^FV0lan|fij?^TV84KuiZa&Z}ME8K9;(9TC`nMAgY;zR2wBKTKvGsA|wyRc6S*JHWy?rF*p2+Q0 zIcK+p-R8S3H~ZwW8OJmq-wF*0T|R3~+RC-BIKPI6h90hLU@_o!4f72=8>U?I_V#gZ zM)~4vcSUxeOt#p3#%583jZod0nGtuYp81^JtX*_;&y}ydvagqJx!FD0`tu)$M{C&U ze0vizcSYun9H)p^D)|=;?sqyd-VOM)cw@t-j_IGGG(*eRJ%F?8avi>=wti&)zt7x^VRw z_A}~d_Ro6xPCmbHpITLBN%WSIHlbIJ+(C*WZ>K1^gqFWL6ji4ww0Y$c9`9w7nm)~n zC}F+o79MfPsy8ourT_h>pLMsmTeKU5>OJZDI=5J702(>5j&=-q5hqx zH9t((#d49V@~evF*YDT)pZs*dT;675VX5OqTXWyTd8hgI+q`FcKkwz$@ZR$|HRV^E zTh~7i{Vo@}ec!)*%kTQEV%GfkM3@Cx){?(B7p{~#8E)9QtWdXL z*1EF~mYmrxrsC0grcXq{#wX2mU6NLllHnmGt%Y8-68qiXtNAX!*x2^+@Qb$_-k)t` zR=PV=a%SWovs0VA!$kV0oY#piW2}w%NPY zX-%KI&PL~UYM?gX$(q8NgsU^>2EXsLY&QN?`N1sF{rH)3uQsmZb@$Jg@pIR$PPzOp z?8lYjtzK?!=24-GxQ+aUZ*CL2?R8u2w&QKj+nKW^S4|ajO|b9gSm854S9s0!=9;}r zRxL^~OBU|%71A;HJpJ-gVph?lhgqR2?`&r+nfX)S_L#rjJVqX2Hkad@@~c^XZ)?+R z)zmS(P@1wZEw1C0Q|G!Gmlc{tFSab4ZMxd@XMD*7F0py9{`_sXbxyi`!*=T4@|G7? zhdoyMmnrWuR1`8cFmkSX^XhL^!qJUDi*Xdj=Yoi?=bKGTgU7EbM5pGU+Iw^s#2~N>U45VfuF#sV5a@H?VQ5Q z(VA-CTttuRe$KP6DZiOl=yF{2(DOg`|QiA)Q&By?M69xqsp3NJ~QVS1r_yJWBsU2*#EtPC}N;U0`vzO+#M)TfbV!ooryxpm>arR7^E6i+u zMFH`TcC`LdzOtavd%~)^gN53UgI{MqI5J!8(b;4FzITM>JP`VFaowXG=dKyPDBafQ zD1Y+F%DV{bG~Fxvb;C0nf>_VoPCE5NdC>}`=0X4bgPub%j+id zAOE%ZYs_};MYwT{wg0(Wl||RiJ&w2TuYXLYoDLIpLN21`MNq!XW!7TO5t9v z`PUC9I99=*(teeyMxVkNoh%ION)h{duhLP zVNqGIYw1&^>pM=a3oPAQeD18R`H^4CL(NPsX{~Ggd-?Kiw`0EVjvq_lGI3Bn)23E* zJnN9y#O9vnlV5D--COnJg_6d9=3TBAS`s%j#Ju@rQvC95|F=_*^Skx@71xQx3*8rq zU-*9Q1m@BJe(#QLZ5C^ug=m^AHs8ve$YI<3bC1-kY=v8HO*`jwoy}RW^Udx#RaqR- z41Y6IRIXflY?Xi3?O&K@uIkqApPaw`ACy$P6y+1=Ya$g@ep0n^#}%%{8C$;bt6uRm z>@B{W@#es}+wzxQ{`@{OCvN|`%6gSv@3~r2k3Iiz*|<{Y!SZs!qe8#txz%!aY?`#s z`g){;zh(TYVLrhs~U0^dA|`NcPAQytw;^{rc3H>mM(kK6yNK&c@(ty=_OF)DuL$wwzrWDY~aU zvvS|U)NR!s%*8*lp1+SfXPMEo^vji^Us?O=nU`yt>-OZ+^+WSo{kM-95lv=$0kI}aYhuoda%=UXv{@lAcbkhIn zpML}#tki#=zFKun`Ss43X0vz1hm_B?f3kY|W~FEBit#HS1f^!1m94$*8G7)!dC~rr zPvi>hN?%o$1^G@nYtXc$z^{3_)$5pjNy~fZe!l4)dGQVRmjjCuPV{B({&l+7G<%nh zk;->A=epiI(W`nke$wkTpSS&8M)G&IcW=9Q@TwiU!L3rYGjyizl@C+(*59paHk%l> z?%B*`a&1mhwrBsm^v{mV>HfR($#Y@f6WmT0=VtDkb?@>&@tK>S)t{J?+}U(8-*9d0 z;o4uHw*0Jl?HKqj+r-iRwn_Z&!fpHy)E=7!BzN58zQ3>V*j*=PuJ~tjPG0o9vc-Sx#d+@U z^Crnw*kvr1Idl1{=Y#Ye8HXoL(boOz_G4Odex%5myGGR#(Z)V&R$27hIhB6Pd$@b? ztaBd@EHYnyAhY=4?)A&K@5HYC+bG!c+;DaFx&)57aaF%>^&e2)@-|xZ)h+$s4}J#M z=!sQy%?XgQ)SQv4fAH%wrZruMS5=mBtxik0vcF1b@>`n|@9!jDnsmIa9P~$_f=Bk4J(lPjvdZ@CMbnXn8Mcd(n`%J>;kE+197eyG*?^eScsM;x=vQ7e|*2X(=3 zQo^tQ`opw4f9Y;e(4$2HA`T!+1_Kh)Q@Qj55{p3V`#|d>gShmAQj7CTi;`1|6>Mzu z1B&vKgHub26{7V6T-+4&LsBbB6!blF6Vp?jptLiT_KfAycdaOK4=zb8Nd@V3H&8GH z#d;8zogHNDAxPBCKmoi!60{uAP{9baY!bvYQZNQDMYXd7tM*Mz$xL+0uTY2vDKIlM zQZTnPisjOG&d)1J%_}KZ0IiDz8=RV#0!m*QDpmm7Inq=B6enW#`n| zx2rVv=H2^WEc%`|CG}6Pv!;r#U)^?@7pzmv0z4MJ{OkPn#5~Dpsm#5JhtoE0+ooIJ zv!>rY=*Qzl_VwZ~)|)+&|Nr>@zfbl5o9%Rm%=Qp|Z|IxyavxR<_{(m-Er$y-J z^u=-S>-QR5{L`rR>-he^Pv8IdTJp4i|KIidTdEbC*|@g;{kXmU_v!jw>t_7_EMNco z{`>uHtN+D>%UPeD%Hq4D&Z)a{`N_~rTjpzJ{?p1x3MhX1Un%pGzt>xt<=nGGGS^)! znzmV8_~epw?!pCu ze@bSpZmT;J%5ru`-J_|^yIU9kpAxg#Jbce1f9}kR*IkXyIvaYr;JD`&JI*KZbi6*L|uK$$Dw}cS_u*-D||^ z+;Yy!F25LVb?RV|$aTwatB!d-T6N4(=eUJU=LEeAKmIIncrEnjgs#t*t^HTJzE54| z627(fv(^9RB`@twyUu&pSat6E=J@^Q+|?Ii9{-sldcO6UmvYtdd0Bs6hOd41FT|?5 ztgcCwYgSlYkcIt0CA+6r^cQii>+rJu|KwTS(k+YT%Q=7ArXK%L{mN(VDMDKDTWp)$ zHce(e>uOW&eZIuh$y?pt_tJ%L7h3L3pA-CRrOC|lC6?}099_H5TADA~c_;T=YsI^} z!gi9|pQ{9FuQv(ab#7{R{}jH@%RTy3&G%lkE~s0QwJxy!+y}S@DxLnUgLQ^11rZTEE?*GIi-__4Ixzyx5D*e5BiKPA4SI;LaUpLKISF}@U z-=){`O%wD$3s(-%-+|7=7Y09ZO|N3%|S>K$yUKCt+jdI~~%Ae+b zF@yW4MNr#SxvMc*bNsU3Ula`}y>Kfz@7^uxbrHGuHc8D?*PMGP{ccbFzuAc&ww(Wd zNhSUFis)?BR!vpAL+vgzrNn*Y?GEL|PIss(QZFp-QGPp-Avj0ovB$|{&n|8;-Su_p zhDwvS9JA+#<$d9Vf9sk{^UZR3XQyxKUzeGE!)9`e&))6HdYRWReD$?0 zEuX_@y@>7m9=10f^X_hzSn_|e*^76IQ(kJ%P1h35oAdj{yIaXBM@{0-%C?`|!ddqu z(q+b;we0!)iN9ldRo{9mud7kn7|POX_xn>NkYnSY{;`)v0wQ?~^>X$JRvSt!@iB6Z77%9b~OozHdkDFyE~Tdem9itAha zM(T#$Mz?(ao8J~rzcD$<^5U+bkI$7l1){z`Z<3SI|8Do($tq9dUF4HrH?p5aKKXVd z`&-Q0jo07o{FS?YW_6D+cGU0Mezx_?j9@!W(fuYzh$d3xo5du%#xhV2RB_jv4Abs>eAx=!hJonBxMSZE%Y{TR@Uzg+V=92 zZ*q#X`?2YAhtnUsOX&nm3bwtzjz|8Ih-&YXofmdR$?Mrq4OzuddSJ0)s`c_@8Ci)Y z#?nhHnOC+fdKOfDBZHsyyq4UJ<3^k26rR|ouR7--*5(}`c2$zLk;6m@2&T63=a z&wlIRyx7QyLlJ`dzuTtX2s?6GvOT1C^}NF#pEcsOA8ftoQLLgoBlNEN@{}1m2k$$@ zY?!vVoYQ;hVq`z733$!lw4ft>!O=a1FWP*w&(=-!ob%7+<3?|xOR33g z|6Tel%U2SiS8j1(-`TA~)`zF=GFW)4VE(`BR<|s|1T`K-Tr#;oZRKO{k6(S~XS_^% z_{zR=%7)7;ESS$owXMuGx%8zm^02|yUblmHPh8>9>5r=mv0+yeIjuZ9{lS$_JN1@s zJG@^0)<(UZKV?==G>P9h%jn$J(}hwMVkSmef*>=Qo_nkbBS;l_*6*Int3$iB^7j)ma zJx4F9s_Xdl#b4fXU%RL&IHgm7p_q{P3t&!PmYg!>1W}P(@8F= zpZ7Vqw7SV&lwEdbqTO#*iZdyDm@=W|@zpUSM-{jJdb z*POy*vM+B}ggMGj(ox>|`^*cgz6&>hpHZBfDLVTSJI5Zws*mgV1u8sy7C*KNJij!_ zs-HE_t7X#38#}swPn#tB&H34^->io=ytLdIAHFgrx8-x= zUxkjy{U#d6H~rkDx-WIIJlm0K`E}e4yEk3dcsp;+{6iXBdfnKh?=A4#X7JWe6Y@ytFx5ba_www;0ZIGG{FfMy? z_oAd_*GX&NKAA$T34d$Vs!nfOaPi_&>$fMSE!Vs}>z8Kn)LPEl8gsXYB)|8VZ8Lq{ z8Sf*LBo95$@n^MJZ&qF{Wi+|%G_R8T+bK;_q3$yNwzsDEaBg2TU&!yB-@XYmRaEBm zWHL;#{F*U&K2Jaz@4?BemNvh3{G9vz$BW0`VmAK0pdZ^gTX1&HbnQapElVdXDvvqg z^J?i1BUN#QXN&B`nqlH*Uwq(Z9M7gIaOWd7Z;nJ1)0~FCSU_&lFl1cIMr1 zRF*s8DbgqWDO7-Mg%Wsps~3NmYt9cy(=k-LoiLwezNY=3DK;)0TR_u7$tw*!P-G z>794MuVa&AA3ji8`dVQ^z&dWjg%uYxa%~mVj!atC6`B(k81vKR;+eMQ&@GQl(r!sS za=YH6Yx$LToweK3E1nD@8(hm{SpyCP-r318^Klf{xtI;>lvZ1&m8l<}pOyRWAA@x% z?{wKIK^b3d^jb@po7R_?&z@Lq#A{hxt@Q5Pl$T{Mo0d;$+N)D#m6v>ekKKkjzw%9% zFsv>Nu?mdQwEK2T(&%`m=ed}teoJqibDEX;m9O1a<)Qd-k2OCZm*f}SVfdOmnRC;M z#lN<__TN_!A177%mg%B{^(_y(hBc~;Ngpq1*t=*s>@J_KCN8x1!mn*RpYZBmSjcjg zox${L;iurgy)7S}2+Xp~hQlB<_*eKEH5!-OXcx4}WJ*ov`t{I@3nGrrql} zo~ONh$2(hd_HzrP0;{JRW_T|xbux1bFRpN#P_jt$@U;ERCC zrAi@zb=_xgAC`HzPDAdrgX;M?R_$yno*3O(&b{Kv%<44zM$7w^dEc&{SkmwAsC>}* zgY?OrC1KC!?oYRW+Y{s0l)m$9kJQ#P%E>l+ zY+I679DjD6_4tJoSEH+0J?0!fFWoz*dtd4|$D7Al=kwcIJr6mh-x8#?&M3Oj_=$nm z^i7i|h-)oNVLzHN>3^PbLfp<%PtB$mabL{jlzJ3xfzL?rfcr^W)aqy(OrnfCF`FS)j%`euhHgA8ruCPLH zgSlJDPtD4m*=BWi3CCkE)J(n5bNyhSljC>my7wDw-u>8n_|xM{H=hMPJNEX{#_b>0 z{@c{M_)PEZrOEl{UxUQc4izoTQ91Sca+==Ky(je-R4ksDSbpS?zD3C!*_!MV7LOJD zgL)Q+9qn6KR~N%o!eeo;G%hVMzL?S1aq+9lGKQspQx{%STArI1yfWm}j!S0d)9ibu zIoz7|y;sfQnx*RHXYJXgMmh&2)$eRziEb-gd+yD8*~YH|Ej+A&Xly2rL}%N6>i-)HXk z-Ynr;yk1LZcIWpk@7WgjsrEXFv`^i+^CFLD@Ryh@nN49&EheQ$9OK!r<$HVJ@g}!O z6@ROxe`oG{Ipv(i3hPP!7Z;slT9N#5i+6+ZI{xL3R=1mXDJ$*}n(W;yws6ylofEFi zj$IgaO!m!o`6)&dRI1J`@!6=Aere~Fe%1VW%$-K%K|k%gao9ayi)Jzf!Qo@m^+tF%dGUFRB;8zwQ|C)sb$^OSPApSJFS zo3gXBgK)9uv|PQNAA-VfPMhu)DdpceBUeWB%1Legi#MHFax6g9g3fHMcQGGMEq}1- z4@2Mf#E`Gz5_T_b^h8hZ3yZmN&&hB56cZy4hPzWw7u>Qjp0y+PxP_If@UGgT^=!wF zc%OV1<1u&Eyv*wQch?xdWx75&bT;CQxANM>Wy;gvN}u03<+96$?HlaXn$DiD@L`!f zzwr5wn}1X+?lcQX22FL-Qw^`bnGv#p)l}tk;p3&>t&Z+lwg1NPqo!udfBnDUQE?#J z_1HhA`C3Nb#Ei~NKP_N!J6Q1Y%cDy(`+w!N@1BGhF?* zW5OPWqB-6>r{pLAQKm zUAl~dj`R2L0?+TAfB#;)(D2B!_lfmP&fhHla`Mb9?>W7;B>(A!zrj84s?VJI&r^Nl zaf|hvM57)K?+nXIX^!6twfnbj`!{<%|GU_~=JESq?ALER)^hLHdpBj<+y2pOdfl#; z?7y94S$sElqnS`^7}Fjj`8OH+>!W`tq{~eCe{RA?>E00YhnM-k#_ZBkZTcjq-n*)f zLq|PawKjJFZlGIrRMp4)`s^wtNH9- z`rWymF(vwc)Y~<0K6m}}lPJ!3uqi$DQe^essRw%9cHMd%yWpT*`HnZu3oicq;eB59 z`n~r>&9`GtrPqsRY`A{ZVR6xfmikYu&bwZ*f8Qtb^KyijQ77ZsxF?BojC8e%7;NQ= zV!v;G!Wws8?FQpkFZGJQOEtB;#Pc0f*|vP#I#1X$LossU)X!74JBqxS#r80*>1Ge_ zVrS>)0WZ8iYp?rpYRLyD$5Ux3@&-;-+YfJCsI^W!+;v$l@2VXyzXrQY+>4vAX`8^u zn>=sbgDMW^Kb|l{BS|LKvviHY8e{R5N5zxpysdSgdgaK4U%4`@-16LY@81|}dR%UP z^r46QdUnZi#lRr1PuE=+rBnsHv1UK>sH!Wcj_u|1!WMIuv*D^& zX9Rz-i~6;!e%6AE3tVS)zb?J~U47}2XF-Y9m-Nz-wr?&A(oeJ7SR2>baxtg;#fO+* zSwA0saMtqJdq=x1YxU+Q6E>gCT{Fq~oY~~EK-S(Vy&fHN^*c1B7~Ol4Ro<&i;J5bv zbHiNpqWwloEf29{S|5|&^R0P1!LM<_n(!d&o&Oi?we0xY8}D&q*`&oQR>Vcd^1NNo zX1V62xs+9rWnWaq>D5*1YMBG`GjT;{fe05fI#nOwrRC=@cCtk6*rL^2u$0_N=vBt;E@}k+?TfRNC z%gaz)STU_}kwdy*dv$2yWOc)-PPW&s&c8U#Z^KD-tDAOQ3d+krpP1Rb?pTT3&2S#W z-7|yM{Mh*RzklhWYeydjB@Zj%)d%cY8i= z`c=ENSw6MoyCc(c@mYaaEwc179%Rb=dGX|g`|NFoaVa|bZ?8I@w4L?Id2v_bzFC`i zKP_U;i> zjJ@3Xg2vl{zu!(zmS^qcwd@OP&g3pWBX@p{*6G5KGXJ<%hqu~p zm#RbWZ?xB+QZw^vxlG*x;lc}-?#*AR)1EvhKR5D%$6lFNMJL`FT$sk%c}Mn6&&nsa z)IVySHuK?=3KVj3-gc~6=}fEY+`E#gN9|r;$=$8 zMIQQFy_mPa`B%fm_{;aN>b?|t5Il2Mo8%7XuPl~*tE|~|lYZULnc}m4wci8p&$rnY z&2h?I({}5FkkO=cr?1<@KYcyA`atK++g4M#?berMR`V9DUw{4joy)$-8w&Z?=bxST z`s>=81>(QfOkQyD#IFrmJSoO{+kRfliQ$-OwRuac@%7iyo1Z7`kF@_Dd2MyG*~t&m z>Gf{WAEaleJ3LbVqc;1a-M86>e~!h8o3urW-;reBxAkYvtouR=A79_x^4#nC-<(?! z)4tf0n=N2gs<#U^J6~cRw&Jkd@}AbU8{gYFXj~3VJ$C!zywFXHMHFLt>*`Lu@_3n5 z@Y1?H`A*Nv6`K-yuYdlrFL>p-)q%Z>R~N)*JXN$&R*;PM0L=UM#G(#Un4Wmn2q^n5$n9o zoL@FRRh`o4eIcVY@ehaL^1a!)inVVNZxoyP6)xDDZ8CMq&kVg1E`!*n-3D_ z&ZS<&)w|+{uSH)SixD%00`O&Lx*qP~?$y6!8IdmeM@ zUHjg9i^{8oW-?#y*yhjOdf?Dbo!C|T^!c|Lhc-6Koa6UQJsp3#dGE!V63aZEq{4`0 z=2>DB?pK7hzId#q@LuNVjxff`^wm%}?KUH-t>r$SAQ;T*p)YJDc4h{hQ^s zr@;%$6VguKzTM;X#W?z&U0KTA^E_qM^(yvbPc$ard|>CUx1Rn_%edqrb8 ztTJy-Z+9z5yW4_+AToV^|I-z~;#A$Ee%UsJi{cZbC#!qPl z=O&)zSI*zjzus1Sg1KdG1up<4SLH)U1MHPDe z7nAy(-43l0`*2C)f!LvIraMkI$sU||V!GSS$dXz$z073ytBk9)UuH#Lb3eY?`;v#~ z^pC!_RW`=^I=RjBA8TYQpFecg%=*oD3(pe!2iaGJH}KaUQ#{WfS?JSbZ}$F#Wzfm3 z1#|amrYLQB*u~rzQj~r-S0?k?m2JD;TNN|zFe%mgd`9xo`Mdk(6f;cQ&DFW;l+g66 zr?1_VpCNT8{`5bU8&@6}UD|fSA^f38llI<|hkgm_XqQS?GWc%zlh~W$xVX^U(Qm2J zhJu+VS3X%Kmoa(ss!tLN&8*}vSXlNMxrzNd87Zn4VV@bY&qm;3%+Iry0Sb?g7%yBj zQ=30Qd1C7kfiqwF8ykH!Mc-T3UuMW@yKll!-OtT)GwzN%lWePZ&heQ2OlvP)|NXVQ z`tz~Wznj$7ypg5XuWilhyti)3 zrhQYw@80TlyZX^{b>7=!{Y`1{+oz=*UboP}IN|g|_8+%T@6}rIc=HsM&r&XXl6X=I zjvWkLdGM*ki`y$zZZ2;vxWC0DN5N!~RWM@_!@bnf12>NSY)bNFJ;CN=f2;Sb-=2%f z(!F2aFhy8@HI&}X`?hVi+)eT1N1N&wzs!;`4XuF){KW63v`V>xvKr-e_BSu4 zwmwL<-|z6&i}#M`)tfStY&L7Js+}2F;Zaj7pW5!#`7ZsiXP@%?+v>69D#7bs?R%Lo zw`8yBhb?;gn`g!Z-emnR?R)w0if{b=+i&ezy4B`h)@-&6UB|`O`mzmeZ#`=?TJk!? zed>2%zaNFKjI*@v%()bMb^D>f^LInvm@nwtvupJV-I6^k;@A9rpY`6UkdZEPqt_&ckNi=*Vg%j+hE^Lc*Gs`C}@=Qwe_$#!OT zd+7z0UKX<>OOE{%opZ2ml|BEvX35C>fV1-Ma1QHRj=z|GH{6mMkq<-QJD79X{>ZEj&f#2lrZ_n&p*ie>-2P?3oW&p*7|CP%JF zX>x6xMx1lj`O1S!e(%1Upqj3ilD_Qo<4ZgDCO?!lmufw>v-{Hxak;BW%U|EWcbI2} z?CyWLiLBqtMND=@<;1$p{#$Bz$?bxMvGJuC8MzIu=d%@VJ^w1zaqd>*qK6+EYU5UD z&bG99ux9lgMV9vYRqXzd`2IWj8x*c;mw&&#@U8N!C#RlI>P#nOEo>e#lGKJogT=zaZ>*FP47F>&7$XSH9qoc z$;H;ZjQSwk{g?ZX32(mLakzN#-;>LGCYA5oc3Z9V+su325q>j2WIvqy?`zJlwn)>D z0U4JJF5fVhVoj^RQX-x3&!u`5=d;SiwpWi#SvV`byP_lBMOmEtc4fDi-&BkBzLxt774*-pf0oNHAN?+D`r3M{ zWv8y>_k`TLw0hl)6w|rpiZR-3k-q8zl3%pTcJwM|He@~7kzjgvrsB5Hh()vNOt#c1 zuHGl}_=G-p?G@JEb+x^5Qy)Hw^kn|B>de8r@_Da?R_(UW$hCNDSS_=qZf4osmkGQX z^%};sYAe3xH|>v4+v;WAqI=`hjOOgsllXIacCfE}HczJN;arm5no?SBTrRRC7^kyDtx4JD8TY9Y(L~VV~fBojOf@RLlR<|#=ZFzhr<+Hx@ zrF-++X59<_vXA>+wPwaXrPEn@Q}sJ`9{aE_b=~H(51p%HJub~OU6NBL5P9y-hGR3e z7g+W^a2NfVVZZixalOcnb(7}K?EXGi?N0B*x;T$ZGaL8!y6qHy)s$`Nt=^jY%I&=Y zmqoZ(#hIIP%kExVwb!~g;#_UTq}JVWY2OtY^87X}?ujeeJH=#`aayQ$c2#?neJaZ=T@%x?TevbeCC#Iv=x_1F0vH-y=8}?TDquZ z@zi~*3T~=*txDKDujB7p<*b-PSMS)q+461Doy22zn~N6*&NMCO_xtVP-J$J#eBXyV zpEvE4V%omh#&h2sHr3;gWLb7a>OH>5$)4R;T6kyMuG(Akt(=Y-{xK52b(Lj$a!`00 z%e8%C9{VQVsWDqL@y?0D3(rj|wl7+=b;1+3rB`FNC+7Ay^G&{-vF-7x4BabjzZF9B zbW5~MPOY5qzHj=suaESmG`UH>XuWoCvpV;Vz*Nq6qH8Py)_r*B)B4UiM)Qc+={qe! zmptM*QyOP@r!sxr(X{uR+2ZC*(NABL*S7WTtNwj?-Ii0GH{35eMJ)A>n|0r3lh{7L zbLN}o{aErwLGXl4{5$T=EZl30Lm#d#+hKm>wuMplau$tCJYUw7{g~+z|3bZU(?*Z` zJsz4RHxw_LpMI40_{4)uzH*z>Iv>O@DA(Mw{kF*~ZXRd#45@Jc_FlJxwYRwfti9Vi zP0vm(e`%D^S7v(I_#w;l#IBCqFg}GDN={lrnGIfWOVOzq|ikuIL_O6$q%GpxN{A3lrDo@%50y0fKF zIQzQaqTRn&Y(D$-o!zT9#k+U+Tm8QL?%m>VqRxh`k^c&W?Jg;rK4R**HJ{bXY+KQ} zrAkd7M9!{Q7|Sd$Eh_8X^qyDxcTOcM2YLo%NQCa0s?4cXw7ox9B6ID6bGZ(i#FQx$0b_!vpifHAkKE;S?VwE#no(Y^mCmKtrX)-6b&c} zoBE+{p2@D4W^229Y%)3Fb7)%81Eo$Tz3T_-4Bd$)-(D(td+%9z zO7@nx-qCvpn&MvX-7UW6%7y~XbMH)2{{EOgvqyPS@xN5P51EBq=jO_EE|gmE^7+w# z8|t^@9hEmmtO&bvtn`)5rFu7YsUREH?k|U3XRV(0L{@W^==m<2Z;=b4KCU_O`O=A1 zawqC%idMd{U9+UNt%NJRlJ}vIs_xcZHq)0KyBOM%5xe}^k_rA3f6up0o=|H0d;aaF zor^A4iArt#^03e3_k3%2*L;TwhdWkENvb{-%0HPO+ICL;v45m%{zQgOZ3od@ws+>6 z1(5P1DX=Es^dm zac10O*vtM|C2slyv%W*UOIyEhx-RM|QMmVUu0-gIo$IQOp9|U0!fJg>xuV)??%T$E zjsHgCGMC@fmORT!td!qgXxGK zPqp2Usbaq?ZOb$FpAVEYpUi2?kN-L4QO@;e8|LI6e{I&ICCU3a$g{NpntnwMeq$$}-b=+G}#^q8DvS|7b2Hv8l-|^9_e- zK`2A@(+y#_Rvdmh)6ZmyU~HYl)#6JvsvWTp6USZ`J&if|Rq)S>n(1@G8C8F0iNwgC+gR{S>2JB@ zf^RW9k{0jGXZ2!NG3=|+_Ug{k+uF|1z2wH8*JlzY-v72Cd~17F{V`sFO{^h%P3(7> zH9T|QCu(}#QD})0ws%mH6b{ zwcz46;Ro6aciAxO2l;*ceDzf7p-h>1)^ju8uj30%j9&hswEm6El#fgGejMxP^-(`u zo2F|nd_`yhyZ_>&EH`u5CfV|Rki5KMW8;GhQ!O`FK8~Cv#52V{^GeK5&UrIDLQ4)u z9u$^-@GmOR?EF{Gc{dLHy;V1VHLL0LLzlluhdf#%>+oDMVdHsjt~E;nPjuC=C+0jb z<(%_R`)t7D7i+$6^D4YqE?T=jUHy2kn-nAefxGw8r}X(g+8m&1_1EmUPmNUagXQn? z{6ppM-i@oOEh+zes(6q4C6BrUTVtg|`&J1v*Q>tYJVW9~#NJXy=KV>AW&$!6Y44v;%o1IpNpUf`1&<>uw>Jc7=$)VF(|hsN$<1H#E><4B`1^R}%s^3cQPIyvqr*El@!41O-kAFOgY&h^YixF%U1~Rb<1L4LlS|Hb^=0U2YwhLhwG!ndG>XNIuC#s(j_1T^nBCYd+?_l(m)hBPvW@dO%(w%q5 zBXUCi_7%q0nT58N3+(-$w$%T3)6t^pjc>c9Ui0SMJr<~Nag&09Jdg8w|# zFD)$G->o)n7ij&EF0YfWwP2RStA`tWW)*lWuL-IO_SG)E@NjwSRI3*`iTfWa@Y(-u zYqmJA4M*d||Lom6|5XQ5%K^#bwHwT&ZqAyuAaNV$&9%m24N@{7Uob()W8W?+Pjpm6v*~ zUiNX8*K)a+9HzVaUv?*M?Eiao!;M!Sy(Uu^1Qt(}ySV=K1NW>wJ^m{nEu2z3QAyDK z*8xu+hs_vlqVp2}^u?p?^Y+%)ku3j#mLR&ivV z{AV^<=vMNV?UfhI$`egEPG0hG_t@ClulwclN0ID(UoY#Xe(*hhJ0s*&+%HRxE%ray z4(FJhx*k34e8u94R+W45g4aY_+cO$Zw-;wlW9j|$^Ozhs)M}+IsFE!=3Z4QFFF_-#wvwbGxi2_v0C_SIY!XG|`e?ziQ{cQ-&u? zGv^n^r&ew+tt|(S9T%P-`|K`yCGy4Pd(LGX+)|tc zEzUcN4Fi6!D+(0RV|(?+SRnkel<|8Bale(%W27R2!}F7$>pU}?9AqbpD$BNNlS1H!zXi9Gp7pwK?~R;yIa2gN z$kUXFuPS}pb<^sWtk~hj^if1gzP3BP)Gz%hgPVuc%Oh+@;xDw+f4aZiWZ%1v-*b9j zpARbj`dUzY=bE~|u~qLCud^OHb$82)r588u-|czPd-3T-i=#F#2&{e5KFeGDK$rHh zT^CpExVKfydA~_~T$*3vZI3Ct->f_LT<7n~bK85BT_2TxH{2BI*1P6s%q^Y$OTTX` zyi!~+{a*18#aIhYskc14r6qHVS3XZzlh@C8entP8ue0Xv)Vig;>`AP1iqCnmo^PVv zZ}Ygfd|WK0y)f=`>6?XBRRK@uL_fZ9-Joyx(Q{^ftLmqQ_h$aSBRu!F$m4JS9D2^r ziv1SL`mONVn_C$ne^+nJDzhwJ$b3rGPX8(I;>{-xKbJHuunf&KjprE zf=6rByiF_gd%3>aqz0Nxi3;sJy@*e;#j51%%u|0Zy?E48de!pgv!ENL9A_l@3PYIQ z-o12P?s@LZ-4{LP^1iK#JihHZ-`~ox2RDBFJh0S#on6xV^J*3M3#a^>lh2s?smAO)tOTv!+_e-nS zeUpeaQ(9#2Hr@7mMUUQZorF7^=AW|*uIs<)zx|?y{gu1jS0&lmeEjbPE}0>|tvEUI z+CGoMM3=cMKW)`Yx_Nu@1%{GaP_borpJ`0jLx9ij0^sD zu;jh4Ygm||NxW)Jdbv!xOy0|?OBoj&qHlU{ThzF0opH5i?!vhTh1chitkJ8kUd2^+uIT_ z-s_-s&vw7=*;{t)kVey3VYuuELT!n~;6)^j2p~3*`{LmY;JrRgzJQ?@ZofjsITJAHtq%ocJ~OL)dc< z_o%tT=2C*yHP!~VZXaHC>&vFu*C&*2opkrSY}JiVn`ZxRPWf}w^5Vae3#y9K1NUf8 zD~{CkIA_tJ!&z6ds5X}K_qHF7>@A(!?`BNtGZo9b9$D?b#G38DPE}~JW^H*)*4ter z65oDKFYR4;&Z)d;&+dr4#~!iFR#kRK_q^iiD3)J-B1H1#$B*qZZHn35j_Fw2q+4t{ zqr<*H^Pc{%>#AbR zKmM1g`QY*5zWVbQ4EPgr4ma?A+hgA|O{qA&Wy+=Nz877;hOgpieKcv8j_1el1@ZOb zFSpxWUSI#S|Ng(%=6{wPJ!1dkef{6J|Ns2*|MTIp_T{+xAG;&m8EgNv#{d64zkaR# zpYQwsfBXJlrta_W|BDlYmIS)x>`z+3>-<-`zjyu>lS6+ehi{s4=z;t>v3d1U1{Z%Z z`+k{zzy9~{{jKpE|NZ)VzpnecSJI*Vem}3j|Nrs!e~!Xw^*_G;{{Q#w`u)*QmgYQB z{M&kR{{D4Ojf+-U`)W2Vd+Prt^^Hj>*DN_Z-wRXfCz(7Ib~=CDBJe`bmy@a6lIPTm zm`r{)tMHwPqPJ93_}t^SvaK?MPRticvVXR!N$r@w?#~&k%RGbaZI3-VX`mWaefI8T zf3Lrj-M#141TEWn$>Mu9$1JDfuLbW~pX^-b-)UK8N*7021w>c!m3+y3fLN~t@< zb2D}4cmI>(Pm%hl& z>@x}1J^R@5_|KWCW&O|R98*90)WAn~-gUgE@`2#-d6)kgLd};C%uga1|AD7QMQe^!;Ow>fR#lPd= z-8DkT)UUrObPn&Ix3h5GQ-zG6zde&Sm+$jhboSL(o8`V29whv6arSSMaX%ZXaB$+@ zxf)X|);&5Y;2)7`a^2_h)8&`LIA(n?_jE4a8n8!Kw|SfP*N1bj7#%pjGyW5=_GG_^ zcV};Z^KwuA^0IE)iz#(4j)~^q&sMP%|35jV(5&r;3b*C;7`rc?o$oAKuU&n%v+U!} zwWc*`J^yaK{!vL7Dei4{dHJ^Fkn^nLo4P7oUMl{(u)OMx zNz`<;rCvRYMbSBDWUJ( zs)Ur2iA%28ExghGZWE8MRnHSyr)x{J?LD4ZSWd93oOe(qj8E0FC(1@M{YJ#;xQ#bY zrhH#wvUfGB?#}JM)SN!Io_w0y`mOnhTg~rcw?DpepC;sAzTWZw(h2{YX-CXoYq($9 z=KGh$5%kzn9-MrP)fcB}NQ<)0Wp%WJESSyV~}C&DVeET?0v*(9RZ5f-cZU39D^t$zh!q@W`FI+mmJ#rSi z<$=S2tG9oWdGPSWJs-Q)vWMn8c(~$Xs`}x>itjdVDK*~1`bh3{ z`-TID4VRY(81wntKcCcYaQUI))vdc9hkUqvSN80ZYs(+b=P_Dmr+YlOt;qZ3+nH;3 z$YA)BBU>J$LFqe`4{DPl|JAe!2Tu#_Q*W z$HCXz?>?IuH(w!R%khS1vkuD-v5MY<$HhC zrIz30_SmjnV*1@;_i6r{)04k^UgP!NW#QKTDSLPO-1@nE`J*}(?YGq#kKUaqe89)? z{BGbq&MNb&ujh-8?mPcx-^{6B?i&30?V82^a?{(I*DUr0>?aDJ?7qmfR{Q7oy+>W< zs~PrfnIjzez;ajm8r3zmJM!~Sr!-D~ux{^Dzx&HR&r@5da4`GA?$z=KGX-ASRJ$(@ z6*^xOaO1IwRZ?utuBE$M=Jl#%OyPKWQ^C-z{N)F=UctL>7b%-%=j!`6y**NKby2+m zQ`^_T?vj_U{1)t$S+Y}K|M-SH^J=+i(;qJkyk+ZD`)88P8ISEuIt@>sN?co;a@|>1 z{e0}D179ETZH{C(e9N&y&Z+P2Y_2e;IiKfvOC9CBcpxISPN=i;Lvr%x$qVlufAeXw z=gyAD67HFIzDO)j_2hY7q&}w`L?6rj*O_>C*_B^!O*TA#^|efD#`7uv+CsiRcz;wO zwbD|o?JH9bAJ^v-%Q@FO)5h-Ge6#J?~4NxDf?>A-TJ*lrsHNw40HR1pWkv#F8O+^8+b}RwU#~In9)69 zpWSrki)oSuEuxpUyUsY_6ruPxSMTYe6X72BXLjp^mWkZF;O7EA} z3l}5IcGVU>IIOYPWqRA2(+9t;(@lRMkmS1Hh~280a??^z-?}}o`o|L$Io_RRL2B&_ zF8<@(W%anmWJ-Ng&IP5+DfLP#cW$Y7TJQZ^{Z#7W+s`*l(A|_%5^bmY%|C&sO?UpJ zQ>~XbJZ1jx^7?tUhU~@<{w|)%f4NHPPUz&!<7v#0=sC3EeAddhpV%!vWtxb~)`Z)o zoYt#v`0H}8UC`oF{`{*>^2-i*uvFiF_YGKNM?8e`0oj{%A5w@3%x2)9IVm zzU)2aYjggRoBGjZ>>q9Z`(ARed#91|R_}d_?o+Rf_b|2l9y`Zrfx6e%c$NyMX)E*h>#V&H6FJ9u zXIW6?>?J=}mlT(n`u&-2zk}!2ZT4H+&Hu#&82#V5Af;}Oqfl)?e(BKdPlZI zzq)65Mo{s1fgJBjxy$UEizP11?Cv|prG6{>{D-jKC!c-z(={KJak8hd&1yS5`&>-# zHM^=WeKl+iCTAb@H5VlHv)gO2SPAGA_p@zzb1bm2S9axA)&&dC7mCaiy}011--gA( zB__`Viam9$@4a}_>y~-bBwXsjcQIYsrP6zdH@s@jXHDmo7nw`+js@P* zSai|;Z0m|iGrY53Z&|u@*X=lg-hYYOFQN^z*52c-Vd}db!WA_q|LG0BmO_J#yNf}D z_{`+ar+eObt!6d)+dgw2Lu$I#uP1ZNLws8fOp$P1`OxyOO>kb4TBe>;puc)6&$UUj z;$Hu((tN!5-h|h=GMRN-q>FOp_utAtu~j*d#k9CM-jOfso8;c~_U2i8mrT!2T~%7T zPxiO`lw-H!|FKU!w!S)c=Yi~fu@BM&)F$mM*?HjYjiA}vx$p2CDzLg$lz;o9!4qND zX&sVZB37o=9C*Xv-7u|9VX7_P@>x&I*2Ny$ptN>1fB9R^olcr^+1JlSvh+oLd$KvR zy3C+`-8BIdDFyH1rDC?0%iFe?9o)QxaYN{bH5(4zSZ_6Jw|Gw!?{c2d0#1cnF?+Ti z{(MWdH^fjW=;M#*WVYQ#3g=bT^Zktt&hymYJvQ-=!U~NHlS4Uov|btag;bsjd7N=2 zJgOr$Ju-Cjnug*pap$h=K3()X>+pGtOO2;rHvhT0X*<(}!*+Kc&v`!g+xCkbn`I>v zvSe1aO?W6~8GP8Ttzgx(s5`IYgw7siFpjn>Sf#gMR&A+=naTAv_XL|~Z7X7)Tlk8% zV)OdyO!Eh4>*_4_YR^BvFzCC7b$^rF>x3n1IOc6N-m-G%lwX$3QM+e#Gd9|<$@beS zZspr^<$QtNXCv-KZjW}(Ilt$U>4x{8YmBz`xfU|C5-`Ue|HdMCFy8q~7%XE2x zk6YWs3x8F)>{NbP@6-Qdl6BP)=j7Ld(!DF)MI@ipUsRTpitfMh%;;-Ju1s+MhU1lA zI(m;?QGT=4zUs)a{*~@GK2?6Qzx z=?B?c@7hZDPIpSGdhJ_e|k#e>pFtqFSlC_s!Y|K1Fr==I*@owM)jRPRC*7UbxeKrBWL$}veCB_^-Rl$f#mra9*y7Exqu`@x$iI0j zo&~*|D6(!%Y4zJRSqAS8-_tL9UT~yD+I^+e!>$>(jMlV16Eew)J+R=pMeMxWZ{;@5 zzpAc!H&XoEv7e3?@6B?U*7tv5O~Nbw-ZO_+SxBAzuc7?mR(A2!7PjyQHtTnvi;@@D ziIFw5c9?l@+6y*~)APB~r%f+N`!OZ{d%ebjEq~v~x^b?WBiG&cMKw*bq$St>N9n&y z8Yh%)ep~#Z%+|yxV#b6Y^?i$(=KOf@t?YwxK5N1>HTqV#Y_$U)@*{{c^i~-GZz=D|2n$ zIWJyx%H!z_Q#lhOD_{RN9)D%r6U&&-wo2S~p8Uk(%aJg)tJZ%~66z;z`hH=>*~RK+ zea$YfE^ho18v8);;>K!)`h@|xhdnqiF-xn9EbNu8?0wM~J9B$qiASY0v*XiTnbJEA ze{3^Ll8nxV9xGVttgmzT#*2@07<-(}l)CGY2y z%_j3bUTQ7m%l8(XF@KWVETttSg;8q{7wLvatv&4L{e|(>x*5UjJ!W%~>NHa?u=K9u zaw|41+!>M}S3GyRccQe~vrm~5^wt~;3_a*!mvi)XG*|w+Q#0>oOzA9rT=iDDEhd34 z<<@(~q<67=i`Q(ZI9T}ht<$XcO^>Y}Wp2@NTztCnVvOpp+pRllB!3EQ{`7;-Ave0{ zI_nSZ>z_Am=bm@#)Ym;ZhKuKgCx1Q=C+?hbG;WVmO0wPk{7fmv0})5R1~vZtleu~K zbrHGy2Ln^1?Upj!yXpVa<8sHjV@(aW*4y~3i8@tMnfKRAxAd8kX_Z06WxMknXO>6Y z+1%9>clykYV-x3ntz{B1E2i-|$*4(3c z{>@2A*Zr?LuI^s0@k_LB$;>R7xzBEUYMwnjxp=9$qG8s>hcl;~oEr7&gY)?s5tCbW zi*{C(&M#nF$0fyT!OJjt-uKwsZYi+^A1#0HTl^~A`|eeSgygT0=L(KK(qFTvj{pUMY?pm$1RX?n7cZlcuA17b#S!(Dp>)XkVzb`4?oshSxdbiZeHs!hI zk!Pv+3AXN<$f86`nSn0Pd~Dqe|u*8mkV)xo%!2Rf)_6H zzqKlD+r<+_dks8U&aU<}I%PS1i~7rAQAvZ}7hZ%M_WbkV^8B^DT_^O?kGwh7-_w8L zd6w!&t3MYW#!PKDlMDp?MNqhBv-4V8La#M3smM@eHQaj&a_%zic_4THLbq|e} z>%CI7b8lg++MoqWD(g?+nlWO_`d;Z(g9Q}kaPc2=Itt+C=r zgZw(atX0c+LipH`fJBpRxE>}NZ^-w)0&!Tg~^p{1= zDjO>7Bvg|EvvSAeMasu7eq3t3yinTR zG4fUD;}c?unhz>V-mf~mp>vX(q?yTeBct4TA7#$egp$nH`Yi@F}}_r zUUYLJKhJETJJTf#9*GKX{yw*NTFJh|tT{K<|8kY?HDO5D;*$R~XOqE?vc(f8u3;)( z8{%)Z^+1gGJvWBK-eUFN__A_e8ngEL9bde%>PzdLiEia_L8@~muJ`Cwmb-O9bmd0# z)vX1tS#J*e96ObpI&(UcYx|-F7sXCrKV@|Np3qs*^M}{`SZKbl|JI$IFM8*P%7jeK zYTvT&joiktePPp93LQ@KY;x-*-)h|Ap^>H=CsD{W*`r#}%f6C|XC4Y8b z`mbTCEc#kD`A@RPQT?5h)uK~xeA?g^KI@J1V(s#U-z-)?zg$%ou-n{#t5Di~>2@Xk zsrzqyvshxjt@N~Xx_L z^iow9dHR2!*5?tO@@2u^FEM%oi#C6%dVFtg;;Y`n+x1=Z%HtX~7m4_uRkvStRxn{* zOvN?3vnOUt^eC1;o5k?g^gUA%+Zu(yLbF+#>ES0>cPuT~=O=Vh;}uj7t?TR-HnBsPp>XW-@z_AEnnj8l!aLrCflp_7VFJq zy`A@G;pVbUHQ~MC)z<~jXHRld2%E51>v*N^$=i3UgVj4N`M+Pj&!pQe{pChd-o|zB ztiOfmo?HF9?`)xWW$FEm{Kd{|(+qpwv4+&mdzJU)vCga8FOJ_UUUzf1PSodz58dpw zuVvP$hAozlH_x!W7w8qNpQU^!@KpPJ`AxHqHh*u=dTS<~8h(L?BgV{_??iz2w-4sV zWj{@LZatf{&?>|-k+))T;Me?Pt#Tz5ZpHl*&K0HqNMGOGZ)?k z-`qa4@qK=3eDzb!7Hy3iXI8uozFEFFQ0>Uwz9VVuwe8a$Cp>ol`XMJgX{s^b-rsY1 z|K-Yr+6r&;DE5~yS5LZpVj}-6;gf|bym#+j%}!Yy`0dmlo;vrox9cL_^jv=5=Jjo@ z2lHOHS0DQLuitvuw4!fLU~4D$qP#yE9`YyMFRfr%yYktkr^fH4oW9+<$g|~};NGpp zi_eR6ZU1)l@QU5(Q)f3`tLhO>Nh{rVTQ9fRb6()8&fC*0Y{akpTX}JEqFa)jvw`eS zhnGChxtD%y{@fI9W#aKz$Yj?MwVdPc-fP}!Ik7WjmH3;}@8&uCebsudt^aj~&HHJ( z+vYSH#+dfL*}hKUaDTQ`u8CIi9GzWav64!z*<4b&Hx4|oeSRr3#w1)k|9!Ocg%_K9 zT0UO7v!0bl$g(Gkp~yV2aEIeQi`$#dg{-hWS9$feh1Q9(>4}^CZPeono#y^sRkTs| zU_al2*tX5aqCLFpiu3eWKihqHwXTzd%Fd4C0pFxFSM-FeKF#05`)=2(56ydQK-A?= zQ?=9Q%QdiHwYa1pS^hn&eSYEdJC=XW+&)pVt>m6q3_!W~cQw08V%59pu5+OWoE zrPE3W8Kb9uzCKr1o%!0e_LjE$1l>7OO-4}{B;5Fv4a??CxqfH*hQgqGJ?}0_N4{)w zleAy*{+p6spUC+gv-hpoS>x^G`@2~+k!{=jYm23OnHJaTavF2}+VcJOv56)ef+q?? z7G%aT+NSIM=&E1jC~WSynEU>6ZG(){>IeDHJlK)u*17Qh`h>e?eOKazxkQgV`ZPIk zmyvw2X02eaEBk#GVcjFKmO?Uo(!D(Wcb@8QPxTaj5O_M*K*~$8sOPL8zxlh4<1vw< z`}RkD&y~^c{hfFBxyOgwyUyFSYGlQ$_Fmc%m}@-Wdew)F;Pa}-9eZUh`?{2TY^Q&j zY%jIF(&)mT^?GOTF4Wx4n()Yfhu5+Nfj7(}>XP(i-5r)*oSYuJnDft)$?2hsId9dL z<{O3I=GnXB&jNFv$N$XgQj$cn@4WLnv-{H#v#v`^1-O@}$5kmQ z>*shY^?Z&E^s9VhrT*jArNr9be?ne7@}4CdyY^qJ?8<-JtzRU|^XOeT!rCD-t$Ob+ z{Tt_H#N&k?zsO=QS5*G3G`&*XXjA37j31iEE?#|gyOn24YD99ZqT274P?M;bu1#n8 z4!O%DZJK_R_xllxQ;+{?y!*CE?&#fl-QHX0J=T~P$n4XzShT!l^O0RkCI$$uI(2WI z@`MY|JQq(YmAiW{ZTbDLCU3YoX54*sJtO{!)%&{MZ_5`=x;b5?*Gq0!_u^a8jZYNy zuI}2Xb1~!2rH#9c3*C>`c}2c@yX1De)!pUo_ol4QSh+XagD1+>dv2lJ(|_KJtys40 z-qo%wD-!thl_@Z{!!qWFGrdE=!7h>9PX5Rk(?Z}@51{)5Im-qAw z?tbX})+*M5f?XRA%>GTto2e zHMu=|?#JL~EG{sHn;dEpJM`QB zTKnx5r5`SFN=F+k|Il>1{0uIraDx_qTP%0-2H8+bWLT z-EhaQg~#R58pZUt_qEpZoxiWO-pu}v(wxoxZ+1$oIpU!Edqq)y!8y)N{jZAtwL9Cb z5|LSTRA=hlDaCpWSsPE~oo7{$o<61c8tZd=b-tU`jKWD%(@U@2yA(R_>P+rm4lCYm zdhqb+CeCwZ3wNtrx4w}PGWB}FRhJtXB8xKPOa*fUi>6+`Vmoc;iEL|mUf#^C@SqE3 zSEDChw~Nh7=qw1Duvd_^>QT_mw<}MDyKb8vnlE}YZTY>ekLFC>zB_Er;iCQ7M=gsP zyGk}K&JyWapO(FzOJm#9Tice;D%URh9oeWDw77y#_)rzL^cbVs`JayOhz`vrxR5QIv?{)T0y0`0#uW<7}kxS)2GUM-W+pD9u zZT_Cw=ceAYxPQFvN!Z%jk5U@UO7TDKzP(J;2%51j>qvk~-s4n#S+*liHMS<3f@GqS zu3o&u&!@$`MvMR3v_mZsp>l6@b6;$@aPIUvXO4!%)bFWmT^U@n=5P3y611n|l>7Vb z(r^ABzwkT$(*zNL7pslRU-NA0wwG7QU#TK)_FU4f^2bCCg$xnC6{lC4@Vq!Dd?Vvt z6T=+!4;OA!GD-IyVw~8wYfDY_%M+>()}KAxcjT$^)P&5RcdeFcPh|)dH!EAR>a=UW zztmjsfORTW>3Ts2!704N6XX3^kB$3QRs9K-KKuBLuf#5MVXmUJR!44YY$_|- z^;+Qe?xQnVJY^~*CpWD3yX287@$YZgo$52Eetzp1}TNhXC9{Dj& zFRIu*@^jp}tLZXVn;&*}R%D30dY0$=^hD(`?Z6so_gzy9jh@7KS-;PEdupvrenh9H zT#B%dWe*oqO`;5cFBj7t1HRx>b2K(2$UF;Ym@!)`Go_vD+PQY^RK4y!dk%iPbo=`0 zTyIG)iS2A<*N&du(p2p|-!Jmp@0Kgyv@%Mvs&~Aache{(rEkGS+jW<&Ub=2)%OP@N zXUWSaMOPy>U*ESYX72oG$&b;o>vrv49%EIhdEe~(9q!jM?(U5Hy!O~$-<~{8`S~~2 z6*qo=)%durX5ZZnGPCl{b<*`0EL8u$D?DXZ&u7z+Ti>_u+f=J>rhBi(Y(l_>SGKp) z=WVUnRX=8iAKmZan1527ziVGne021gH;?1_KtKsbD@bd4M8>YX@d*0Jl7|L0f@rm2cURP$qT&+2qq}*R# zezS0Xdxc1tMt$tfvR6evcWruGdhkQ^uf1oBUC$MIzl*selRM|Q`>Q({PMbHpD~tQ| zV^QLpozKKglb#2i*OxlxH-k6ceCynoT-U9xZ{PHt`_}I#zrwy5o;gsao6>13UT{(M zr_FPHx4S+f&PpFvX>7gdkt>;=ELN{qVizfqqb~fT`@7kBt#bF5&-VsgW7)THp-T9< zzubqmeO}u*bGixFYq_fyQHS>%KIFIdI#{3RZW=jzy8oSX(!KgJtGjk@P(Aj+LhohX z$K87aybiw4nsZ$)`)7^dle|LhJ7xQHZvC%Yu6BF*glV;=0UPs+US9an{#)rneElt+ zcijOOMBFdD$l4Vz(Du`5t=)FT%I$wMld|7e_lh65bx!=>w+&m%nR5N7`QPzmUDq}9 zx?0ZN=ihHXzc%yLtK#f>?}iUM72Z6{GpcHj-g)D@Olj@vC4qDEF7lR}l|SS?Fe&(K zz~0J*x4ETE#Y=vNttw{7UQC-diu?GOql6 zHE6FgznQ(q=U9!h&u%79I`n5BJ?EqMc3b3|QpNt7%3o(6aTzcivME)LWz>$@`RvvpbPwr%0hK%0K?x{!SaZ^K3PH~TQC@9uxIYy0lU;cr9N{B(0NFYXdxj+pDR zYv%e7PngP$>zjIhuU1)ae6B=v3rAAzPn`P;RPS_)IKV#En~bQ`!eDBPM!(2;vv;; z#W!aE3;D6p{L9J@$@xcuA0FQq9kZ|9^mngYW9+X`_G|Zak6r06)@u!FO#eEiuWaY@ z(+(TTo9=!(Y1{N>YI;HE-pB)o*9fgY=XXH6ejfK-uf=gwBhC9#KQvEP`grD6uwhD@ zz4NvbuN@oj-Hr*$WKUaWnlFFBGr97hY}uu^tlRTHKAyT=-txs)lO4Mk#_VhFK49hD z9v<4a*ebJS_2m<8i}!qTSSfS*lHbqWHT|`JZ)oagrfgvBn!eztW=j0C+s_Z^p1=M- zJa?uDiHccO-t$0>5zWMkgKQ5LT|iRyQBDQH`j9Gz@4iL@=S{6 zRE58Kul9$jlr4NuowU2ERGI1YW0LnnE#_{mUwzLyTfV&YY~G=m9j|P~?-X3~OG}+< z5>&3Rxv127Tcz8ghp)#Bsl1^RAz z>)W$<;qH&dH=n1U|GMSNrv8Y&pP#sIWuMoKt8nh{_Wv8J1+?K zS!-L{EY59`aW5_O3zs&EJ{Kc9|Fz&PqbIL^G`T(dRaE+Csl$zqTUYX3pZsLm*U%8n z|1O(j#=)%6H2KPPeevhjAM&PcaQb%cR-jMpr9Dri)=k)Zn7?af@rt|qPNg2xmdSr_ zop9i*jpU|7Dv=IZ%ftl=H01PS67Sc}*`YaOhP&myOXszjI3CXNRM&nbs`)%;S#8pc zrrPfl9A+%9U=!Wba4mNE?>)?M22U)E=J?NFJH`9sIm4IiLB{ud;O*VVV1z1=rv*GyB{>Zk+I*joxAi`^2e<_ z6$+;|^}HcBA3>m#q;wQY&uvbQJtp_*wd>#VJSbnlnXfZrM2B zO*_7dcTS!`pI2G?%X02@y>6Bn@5N&7x=0-C+%3Ry*F|e#&}ka96T@%hIc#PDff>oswmDZd|nf zTGBU((;VJcZx_E?ea7|6o6oEg6&=t;M4bS z^j$e0YE4W(Eh+c-P-P2$Z&TcnRmUn1XT*o~Yd$|)wduIy|Bb1&)(-FXXsx+@o^iYU zXZieXb2t4@Vpy~{$}#`3NA}XYYZtr?K9M`sxx8G&VZv3eTMyf(C~kVZSX*|>gWi=p z9#1eb`DIh?eMt7j)rb#F)$TWsty)^0{ai2Y(dKnMIx|ewJ9Wb5O_p(0w)oYWd}GE$ z%lWRxi@R3e$zAYPT=c4_wz`g0_b)TU;%6XY+ST`A7r)KBUbww$TfF)K<1nFxd&B*O zjeoN)-oHP}=JxBXEzegU-CX;h;e3SdytmejFTSQ`+>m(MV|L-@Z7v0iMG6z2F-Bf1 zQj2X8)V=Cf5+3@y_1w1dMW(Y>tADpXn0wBvz@Ke5$&AFzZIG&L207)t<0^JHvWrL#9ygVT})YPIEKT^;*i!AKDrAb3eT6d*Za1snNsS zJN}ViG8MowU zkLOVx!qr+;*&<-@aXa?ikniylv;LrL{}L%4H<96?c_0y_l97f7832 z_w>p|-R#oD{(02EjT5*wDjw{dCnm{ zZwkxaPpgyGx~;S(<=C<(@lVc*ZCv2@=Cma~_-3(!ZRZ(L>4$$?GS7%JMlJ}xuwvyrgWle0|JpKh%D5t*&(VCf zp*dFEM9Rl6cx}Kr<35&?+m{^+JpJnFDX*>NO}90#f8L}mRiWSZKchgyy-)cm^Ly6q zonfB}8y@9H&CXuWBfa}C??E2nuRpis-OE^#H9zmd4TFyp{W+d3oOt~u=Ug7)%|Eup z+{=hNGx6Zn2XhM<+!GCUD$i%zYAau?`6061ukO+w(SwnNg6VTEuFLw+$Fnf7=uOIu zAAxy0&uo;LEuV5}T}0nm{+wRL%#a4v7}3->dmkw9h$qUZg-v}cXnD@t_Kd>uO2*sM z4frIrZnrlZ9GHB&$W0*aSa8cK$*|p72DhRQCRp^@=bBuyW0L55>@VHR@|^c*-@C-6 zHv=orf96ZRsl)ty*)-L|Jd8TuKR-KBnDn2cYkJT!I?r)32LR5CibMg zj9s-;CjPN{WjU3r@=9p^ow`{8JJP-y7KPpWef9W>d!N)FJ@iOFv-FO=Wb(Rd+lE7l6~t!f~v1;PiTpGaJ25;%?l4?469=`bY|Y@*s#8;)lRZo+j+rW zCiA;n-}6Qb+Fw*yq?j?K=0VsDr?V5P8C4QH-vxKH1Ydh2Tc*aOd~|XG_fIY1kPXjha@*IAX1FS1M`b{=5~kkYF+7Oa`~ zL`v_s7|d48RcpB!n_#SXM$AU$Ys9n0n>?D!sw%&lOQkv;5P5ih{aOA+axqc0Gh{g| zULB14o2w@(sW>m%`dFaoo1155=vi+t+9AK{Fy8_znJ48p9QQv?b-eM&apK~_b4!h@ zUWfLlNxM&d{WI%pnlsCiQ!I{)C&tD1-p!fXxaxQCg$!<`xOM+?WilO?EP8gywDwZ2 z?2K%G)wq52)%$}KC5w0tMuzWw%+pu%fIHP<@?m}-jy7YpE4gXpT9fV zKTW>k^Ew4-WnH&&i42V%xuP44V#`Cmi=O^aT=cGWv*>{s4ymo84w;2@$J~^a#U7S5 z=9`?lvLp6^wEd-&h3i@8{&;k9`GkKHcN#DItFqu+ZEE-~W4;MyZ|>D))Sa<7o3HfU zettcdk*$sildh`Pt4_gbk_#?yBwuB&@VIiFv(`S+O}h7w%!bFTyPi8Qwqt*)XZ85P zg5Avbdvo=^YO+MB%=zQ-T<%xr#<@Q)y!d9kr0Hz2%+2KjXP-~EPMPg(WfmB6=P)=S44U{`6Mse8y=>KLfvS+o#Vn>N-46=k*TT1184}JnM8OO3nL_xUHONZ;Xe% zZFz`jrQ6Qc$s(;h``%wSI~6^{X8xzkCm!6cx_2y4@k3)AgZgnTmV5f_t8B%+4ivU- z&XxEnqjb~CE&uq=H=iu>rv;q3T6%8lo;PcLhn=zc&r`a0<=dEV?QXVWhVi)z&7`Jo zC~TfCC-wBDn~KAQ7a0cwieFACSE=Ku3)#Z(jMJt}y4S0cUE}qVi<<8lET%qNH`ne~ zk>1=-d=0|!tJ~d{?okeveb%WwUr_JD#bf=i)|={Gn5aCTv${cEt@!PllyLydm)zw_#-`van+0xH>OK~PN?VoL|DKjn6ciQL2>#`ZYt}a+&EN&|mIWH}A zfqlHIM6TiDAOExty|{2-w(RWR3!khh)BBTtKjlg88m6XgH33z|orNL>tEa4PSn&75 zq7DB}Z7X}W>ZS6&e5n#wH}>?P+tH1`=5W_N?qKhHfA~LJ(WC0M>*Tl`u6ixokRLwR zV~T)z0OP6y{(f@jJ!<-2?rHSg^;`Mu)n6ZeWD2h2Jh=C*L+r$fHFD4Qdr3v)oxFR= zP3%F(bM0kWKa`o5ESo%k`kNmSRbTrQ53jRmJYi?wJ8xQ8?cC{)6+d6841KZm{>>NX zESj&yEvjafKIgB0{8s&&U#9o_*5-SBecw2dtMrGIYP^(W&^?}&d9r@zK13XI*>3i6 zCHJSZ9g1<%y-gvC>w^5|O=T2W9~CRd zD=8DxetM#!NW{9sTQ7R#UlI>`dV-y6jf{I`*Sjs>decP(GVRZOy~20Rk6pv+3QidZclfw%U99Y;WkGE{rkiDn)iyXyuVTW z_YZ~Cy@fGHXJ+g0TsG^L;}?H= ztv@Bje^Xbijo$sr)-9`2V0uWa;ky&vwG|gi6I45IPoLr}tGSzK_ zZn>iI(7jyoU+>+~dooM^%f#A%i(xq(g(XMVmzhr5+OO#Dob_!@%0A)>9<*3U)D#Zpz{pgU*5 z{Ac@$a<=q5n!7%Av&;9H&1W~w^1k_~@{M5is^fASE!ljQ@9vk$D&1OfPp9|ZuUX2^ z4A(wBH>E7>oW{S;UN6cRO~Qr4LslKoIH8hZV)pu6(>DHNMyAic9J%9Red6ODk^Hwz z9(&8<4H$Mj^EvsneI9d=!Z~rD)W!?DSbE(GdiyI3OSUsuC0w?b>NVl`weP35`qNFl zFY0_6_Hb#=Px`lfzE1DAO-}bO%~+=P*EP=8r0ml%vpybShuKeBy4u~^lk=WjHJiRA z>r?pV?Uj1*Xa6k>JSkm%^nUNNrQA`^pYW!~a@;IPYTK;0i!B<$0@t&Low4AWz8ziu)$V(s1LwrHMrY4D72ZG&{Z z5HBHeKqqY z^xBGdZ7G!cVb-VB$#Z%6yiXGqb@j^cbgLf@eOFP|FTZZf(UStVEjDF|u8GVm+VF8r zLR@mG`iAXEp)T{TE_YnG_Q^RGR`c#~AGW%H^$WH>yjwQIZnf9O+R4*j3eWzowBe=k zx$wv8>7jMCEB5+xyj{BNU;LNX|Cw3KUmIiYmI7TXNW^X_3o~=nJ4LtJzFn2OGxz^< z&2!u(=|69`1$kWZsoT!;h0#k`W66ate-}>LJ5ASn+BeJ8>D-59<8vb?uYdSp^%B17|4-#>G$bG176141x_y1` z{y%@$@Bj1e{PpYc|Jg6FOn)6-`tx+Grt+%&C3h?))$E-&Q90s+<+p>=)_=Xstu9sH z*!l0nZT^I`)- zV~Oc;hRYoig^oYIti5H!L(i!PSq=!FGP^1->9Hs7<*HKW$re79Qy-a|Brmi7yX?e) zrM!3aoKq(yv4LUS@KxKilR4%{__{|OqKXKVgD_OQ+59~@G~0O zZTTOt$i3@#NB4RD-vSFBIj2iWeB=3KYk1_iK%e1G{;!+`FAfMwC)rPVG4YRG(Sr>6 zpo|AMoUM&ijxqEZ@8s>}On7lXQ2tiE#>Mhag-;F$N?Tn%urdDUi3t+S7U!OrnW!}^ zWOlE~_`lHQ{>kGLncZ_<_%I)ycPTMjPQu-S{nRuATz2feGfVxLVBgv&b|z{clJez# zq}fk-Vf{1i%~|IAT)aHhE!oeG2*^8Id=J$-^u+Fp`hkVa?l~QQ0~Wb|`hB8tajnO~ z#^mqUCn_4U738RXUuek4BWv-<`8U7nFF(6J8H*&aPXkU*RJ3a==s8)t!oR_pz3+y= z?}aY+^UGu`3%2lUniRazl#>)n*5p##j@1B*viE&FordpOCf~j=gQ1O2=HtS@?LxKc z^6q>x8y6b$X1|DkaQ@;MeYxIhllsfyz43N-{}XQf-@dipOy_Q^QRn5Hr*2z5>)+^3 zY}UMUVPD+NV>ySnTz^`;*kHTp+FN_{a+c^W4xifR{q(Dc`Z9ixqFta1mVN`qH^{hURKdM>+RGP z2Yy-4c*Y{9IFGH`v^7_C()~oGcRky*=N)rwSSxd8YR7^nHf&r+1XUCcRIa{w>-5FL zUPo3(2=#0*Tqos|<UjHW}-U5|E9jVV=+^e)8}hdssTD{ zcV(;%KlN0!OkiUS>oTU8XGM*1g)(jW;k#Mn4u*BFf0es;lH>XvIj^SHFoq{y*JfhV zQ@znSO|4~Xi}fX&Xvv-)hN{C+pJfu%%4fcwSy(@Hmv-8=d6H`;#(dk!Vk_v;Huc7b z7#=n6s+I}G&DKTf-P{6KygbZIMR#PqEOA}+E!a*j>~!6(vw0Kx{;#+j+a>j&EMui$ zL%gL!jn8@3!_L>&9WuM;Six{7{l=o$LVt<8MNP7N-@?OpaJSw5&S>r$ekWUR+XD$z z{*}2k>dR)xL>Ny!$T6Q~O5grnpDsNOPcW6}S+Mg!%+8x8&DTDja}7Qtnqr}|ob_`W z_pWWar@z=R^z?(eZ^h-pfAb}VdDsmE?+mTSvZUon`iS$!{3+HQg1X}(YLEy`C9%uKInnH+K? z=F;xMB)x|d8j~Itb6#g%GxPNEjB~7AM@_fOJCsdrVAP)4|J=Y=i}#@#<5}JC$#NTJ zye_!?Wvj-4@C2_O&BF|0#TpmyNM#u8mhmZ)s!CZn>xoIh!5&JH%u@|ixv`^O?qpW^A4 z8jC}(AC2MT{?6;HQh$l7^WCLIWyZhU6I^Ef?fj=JQFys(&()6$xNY3NahKvk6&w0L%vrN2Dt&bHP)(v78T${BnW^YW=V({tynOzwoENv!Z9cDx>d{> z-uE*@);?uA$}fJFPq`^t!c_liq)|)opRBs;%aj<{q93nXnp;t^!{|WNk}VGf&3VIA zwr+YaUl;!P-^)hZX1N>os__RWDc;L^cgug%j`PL)n5$d#_0kQ>rP?2wSFzr!{&4<) zF>?z`8@tZIX_AZvwZ509zGs-RZy9H7PP$$u6aQ{Q3Bl{1dlx@kJ@vxjeYehCRs8aN zk@5SRA`kaHtFU;^nVixyFDKqgbFSGWzIlHo|ICeS?*1KknoZ<>WxVj|w1XdZ9N(0q z*3)5pU}l+2TeOE_v)=Is>l@XNS=)Ohxa6>8y!*(-bol2fg{5-q6%5`d3I5)4fBTkW zMYR`h%rdDcbGUFrYNgAj7$4mxsr|tXY&;e$JCokDc09jcE*AcCVTD|G_KD|*-L9*s z7}#FsT@})`we?Wx-bq{Q54AqywiZ7W(q#Q7>*^d1-=&EkUl&(yiGO4#UcSA$BO!i6 zu+r9LwZh)fFFBPZOp-eko+n+p<~vo_pQW_h!NyO`^4Zu+W? za!MZ(g9D8eyVb%Cu14%Ru;j$akYvH_Y|Rr_hM4R;v`^&DL$<4@p9`f4thiSen*VWM z(aK*{M_}X{6&#@bC&F2DtBI~&RELH@=~T={wuRU#IGx$Ia5&i%+zR=jSX|e9Aw|Z8D28pK0Q`!WF6#wtPp$ zANtC?Qg7P1c9G)D+ZtZ)jNL9y;=67Az<*!Nx0|=DA873}YJWK0>e#;ZwZi z7hFj%?caVV{-hv-&Cb+j!N3Dg*Xl^@6P>nwk)7q6##laOEoVc)oB0y5{ypb)uWb3W zrf!zgJr)hWJ64*#qWc6SO#bfXTzq;%LfMAd>b~0gO_dK;q~z;`+573u_!Jo!U8@^* zWm{%wuu9hIbFPcFy^2b>@-4~WQACoK(Xv|I9~+N!CG@q+d+Xel7EGG@eO~vOeKYnv zK6^5uf@SY>rNgP~s*lWiAn1B6Q!nAx;iefT;yov-^50&KFjSI0yIDqh#oA}OMM5u{ zWo}%|lwosTnRi?2-F!=)ut2k5n+=oW^`7|JIEJ)O6JN1_*Z;x7SRMf*n@hfUKz->cTp?&P42-C!yi;C4=FNjQ9eLL%8!sS&ru9!NlGQPBCp3It8Ii@DB zS=8>>%J7RPI;ZQ-Sg!Q=+ex$clH2>&&*u5b{Pm9ad#6nq^GfIb%8M(V%hGl|u#GJ^ zG`L0bb*;OPS7?dk-w?jyo6~FjmVeH=&BFO^R=L^1ri-~uOvT$c#TxR2btDSelbvQ9 z{;-?NIpA)d!G{H+=MsO!-qX!_ZzfSU-G6q&fsS&ye&2)-ePZ84x9>bBw(az?2^SB& zn;771mys0zJ4#kJXZyy|Z5zsGOm2)T;8anazKdkx>5HIlVTL z#p-p2U<{AjohKLXUu)m@N{~5J&m#MK$?ApAr+$syp~J*<`155+-6<)jH`x5lbhcyF zopQ~mY6IVbXfd{UUU^@aU}cvr8n^q};zPynUHZ)^QfMyq(Vw!cqQe6)=u5-U^pSl#4&Vv@e+)1;kCIJti*TZeK9tijWhGdAb)>h^NiS+?nxW<4n28W$FsQm zW^=cl+~JO$C6_Ip%6EO}oXRg*x2@^GgvUC!ubg+S-*7e|MUc^MOWGzoQNb_l&Q1m_ z%T7E#d|zO}%yS=3e25c0?#_1laBRb3`JV!JV;hpsw7Qnso>(h?Sd>HE;LfwN=Rc=2 z^M`R1%(yJwRJ&X5zCerAnWu-oGk&#to1E0Ncx}|$yTXS2XB=F%s-C%`7rfq$Uo8K_ zvbbJGKHtt|ajx8VVvk-~eRo&i*%*$Rth*LG++wahs|+M0Cm)_tvtLHs_RgcDGxB<+ zzhS_%&8mmva!)3-yzF^8P1VX< zQRV&7m_u_|-|HN|rE~9h@auX1u2;sZGNt&s9(k)Nc;UM3!`1HdzwdMj;agm7I-iLGVtLN>HJtj0c@!e{pGlFjn``RxvS3mapT`6vK zWJTWHuWZ*JNS1hiJiF&kvE%%NcEKVy@xGN-dh@t~Tg7~~No@$7yVo?T(sjFh`M#&8 zS&Nt6b$cp$oX0G z>-XkVv}9jYzIwMKqGCt3p?s&bN%E3i;wlq%R%|?9qvH2rkMVRhC>Pho)=zYqna<-o4-OlB`veAwy?-^e1YFK^X1IKoj)%V(^SDqK$;5@B% zy2_d=|KE~NmKAgv&t7<1{cKL#u4S6%bK0u>s;94bxb0=l!+)B;E5-9tX6h~47bj{R z^fcqqL9;-awwEjJsb%db(|W&|eagCn>1!L#RerntGuX6(jk{-|Aj|QYQ5QM=jJGbz zJ2WRP%T-zb3Q~qNZ^X1ICQn%ur zA4KQXeq9{Qbk2KgO2mcuyW6k7zrRyudGs4!N4LwTd~4@!bG@upUpw1Wz*pPzYegWx z`;wKZTc%gfnYt{D{qbidv*|0m53u|*STZ}VHTHepLYDp&Zx0+Ox?LfDFz${1CFv~^ z#jblkEIwJat9nPr)Bth4FSq--U+sO}$@biRbMj5|v&wVssW(16ulwwBAFJ~1e%ste zPPvH+-HZLT9F=9f>%V)dX8g#$S8}&Dh*R!e&7!#1v3wjn>9YE(+Co!O^e)!&eBo%X z6fP(zP5yY>@6Fy<^FJqx^H$BvJt#AO){E5}=FJkA@!rPt=fspe@o!>h(=VsH*sS8; zozQ2eGf^(w=krWX$Ah&IUoyJ-*=N3~%X{DDbVBrKw!gXXrkkz&-81(*;+LMO;gvIU z&tVVO%DZy&9gW2fUQJp#H;(zZi-vUV!o$;7TZ&#gP*|(@>WSQ6|7TVfM`Hg=&q)sM zp66_SV&#kXty!4|M5%^mue3#)fM7QCs~KGVH> z^}NHcP5oRI2=k^I<>6nMrT^XzI%$vXaR@?E< zJFCQH@@IvuEI)Yj>ZXD-8~>brzv|_LD(SkN`(8-K&U|i;ib_E*C#h`aN#C z-@BQ&V=a^{ud_()n`-#+*`;-wt{0=ioSFmq}Ji`&{G%3oGYtzhYo>U2-Ik+jb4I7{34R>}Wzerg7HtebYH)l6(W zP;j0-C;WZ3QKHm|6-e>@X32_U2N`+H*5@n9LM? zvac|+PHEclr7s2F_Wv^WHrr)yoOtfv8KJrF&g+QmvOmmP^eO$z&T00G^B&HdyxPp; zzs~7%(+rNKn=3ZC@9C(YT$h&fqctnOoax{GV$JtFr`EZToa6Ggyf;1Z@@~s1Pe!3| zmd}3oLQd?R`@tYTbgj|64`RWJ<>^b_EO+|Gcxcn%P5J3*dEbw9^Ym9-G2YGk^7Osg z#zi*ApY_~}EMD8^w5uk}U2$&tnbWM4r;FD2rUJ+WKpr|4Toumu4q&tW;L-il|dh5OHMwp$Sw`EO-RW;^UQ+aO&!w{Y$4jGEv*8!y%=%{=ZVz3AtL zncE`LrR!?b@(&hBum0X9VmX% znLh7PGJMe)D5H1&w&(W+8@HE5%U#)b&GX{gM}GuAzx)|9e;uDwwN%mT z;-jxKPxyZL@LOVbWuM8FrpjP%$?uzg7e^;}aT=FKT2} z7Aao8(&7J<2ZC##e=tj{sQAISfAv(u5AF#r9t@AI>*jaItG)awB79KUuJ%jFO(7el zpD*5L#Wp>^tGvG|@I(H6rsS*k<(xV~U%V3z%IGsHNqiL&;LeL@vKKm2%gJ-H<{O{F zrNHX{dJ>iFHm0ka zn)u;$)h+wwUVD6&68ZkGT~GFcMEw6k`zyNt!{1)iZ(dUJY3GAu6E2iKdsY#7ernuW zp80;dHyMvt*93PS;P`(kxv_kKfsqpbHdh^K+cOP^O#`y#r#vsT6QJN8k5Vh z4NcomQ^Ebs99{8+h;+W8K^k>M~NCW??cbY1vcN8Wnzl|1) z%TKg9B7AYy&z6bX4fT8X6i8QcyGc|=L^%Gi`%?M9s4oA>&iSX7g>EX}px^MnXXJ0 zIl!7+xaW&u*rF-cwb=)*sC+na#CNx~mw?Tk1xMyyEPHy`x*_)8xsxw~6OFgcn=F_< zVMYaH9QFp+q7q}L=?dG@-i23G0C zU!$~}*D{$*Px>5s>F-_LruOJ3rg!e2zI0yfTV!!we5qjGA^(qG`Om-JqkcoO_i)Fw zmD3LtuTnhy_jjz-`x)CLU&=duet0-qrvJ~XX-guPbZl5{eEe{*i{ah-!p0fLrqBBO zSoBv_V*Ry`AGvDxZm`e3;h3_2TiNz>(`l<8R7bKMp1#_x(S1W}D$~7d+A9|_=e-hr z!z_4cUAT&ml%#g7@UwgGZreof6z6K0-06^T=`N#lx(xTVqLsUo#O5kpxP1S)(3b>L z$NIy5A3nHk@VumKy)jXwI=!2F!-Ll`vu9m;XCC?J*X7?k-}@;_1k+q(1)GlUmD zH4>TjQ?;LE`}Bvip1M1kIi4#|2(UMkIPuiosgBp~#N*>{FTapKAO74nwtTwTMnw)D zjt%C)W*L4j*j+xaNG_cA+@L)n|J8h7are15p8fmB!MyP2hsZl01=*U{KYRCn$Hbj3 zg<-q*uU&Sz`QW{exu#5qWf=ojhI<~33g-Fd*5)2!)p%-$Z~k1llDD_knfpz=^RdNo zI)B}XT$@CNiX+Z~_4TX5;=j*WfBeg}=ZE^F`sV6?jEv!5W^TRw&X;x9V|iUa6ii>_ zR~#{2@m9j&KOs~3XMI@Azo~Ee><_JTW#i_ynlkLQZarrm7Rt(3ykeDlv9p%v1n zI&XJ^vqaE)=W3q)%8#8p)5UKH1lYK3lc;k)+J4TY;7!~Uy>mU)AXUjs=ZnSN_t~D0 z-LTMa-}9Z_%J5|l9IUOK;fW~-7ymYCNX;lkwiI#UcUpK*G* zJUewE&y1O^Lh2JLR``o`JqX+!aLntB&J;%D%>iu{X<}UuE^Z2FNipSK>TkL^;1Q3| z|1T$=~z*w@XqNemT2&Qvln%nXN`QW<_c=ubtUy^kY_}Mw9zl zr)L>eax{xO;P0TB%zD(?uJKL2FdpG`n#P0jP!g6i> z+{66++n#9joN#*R?z%Bq=BTnyzWJ{Qff670EaWxrJ@i56%ICV?OP|fSr)vJ#?)yXW zDSy<57LKb;>IUlE^SazNKX__1L+lT~tFg`x#o7H*A4066#7^Bx!$#d-ny4IMi6F+J(7YdBrWeZTbo8ju(y9`+ut#EUJ9_xm1+v zaPOfD$Hc$BhTjuNSNq%~lj~Z|6W<5CzZNp>&neTWm`Ic4#diw5UzuDtkUAHU zbc{bNaZ2~0JsYoWkkw@TzfaM6+XHvqo1U$SC;et@R$cv0HtFQ2(u(Ah|2)jLt6H~w zP7E&Fcy7O2rx>?=)Bd6vI(-V$6LKGWt}gVc-XV2cN@CHmZyf8Y(sO?%et75plmC#0 zoAQ*0mRsIGE#`DSqi#4o=VSEIm`&rsOxr++0`B zJg>Sx?XkPK{>hy`dA_IbdS~W(;KJVpvO!n3b^Gs|cId+LP~ z_bXrI{PBd<($hfi1ZVL+h1Fqe-1I#3_Lz%5>1Qd9pD;uB$D6k+PN&`N>o)vo`#ihm z{LFV;g5Zcwx7qCsIkENGJc-_2-XC9QH@*!jbUiTZ`=iQiJ)f(Oz8}_* z`*(J80PDQq)yMyxwLN9M-^oz*`QDRZMTJa4 z%;#3N<#DT9y4Gf*!QAx|-oFlvjsdC(U?;qx^GQk(Op8wDEJ(52NbFqk_fr5bo z5vxVaElrKM?CcbZONvqxbGa(!tc`tpDs^}6Kl|{H_Z-6h{AFA^p}0zIe>C40CNE)* zB^SQ@TR7?M)*SC?-*(?VJ7My{#k$_xbnG{!=>JZAa9&>Lk9zp`uk!W(Ki;?Bpc;8y z>fgut_W%CA-oN4Fsf`!=_WwC}J7mJWx`*@Ye}4}Df7<@vugCrMX7TlZLJHW!v` z`aV_qZ2o@kOScokiyoU7{rG-j>e;?P>lsS~css;}&T;W2A z`yAV@`%!#SZ+)A1rc{4$Z;g^}cfO+kxaY1|Iqx$Vv}U^WUd|4uDi|9!;3T0UH|=H}HR#kixpdyh?gSa>l< zIr*i%xLI(@;wM3>|DT<(nEZa?$A7|)gUmQ>45h>GrcS=?s~t3}=H`u2Z^;6V7l)7JA*8k5)FEsQ!OYPDu z{$(339ANmhRpRo ziN0kuh@AUf;PKQm?`AMXHYomF-4?K=l3((dlF6F&JhhRV_MN!Jc{G07g@D=oqQAOQ zc>g(N9tg`0dTuhSC^G*0PK90WAEd9ze_!m$T*u51d+C`;(|wI?BEH`_>JA4>-4f4H z(TpthOK#W~<9SP}XT$6pIjdG#pV{rZ>UXEXTZ5kEzL^UdCl>2y%@@*3?hb8dUuG*L zb&P%4UKOUwx>vJPjNa!>VqV4=QYe2cbHnp_Q@3B+#+xVfX9>$gH;Kt@Oa=u>W^YxF z?9=(NJY!RQX2l7<*jBbWb>5@R`wq3F?f&Xon95MyUwYYkQfZFzqf>L+WQ`liT;FHYj1C<5kFie{70vwwk5#fjdL;E z#zil66sGwd<`(#U?t+Pcb=$LMjWR9Yp3{mE3$x#=FFl#_Ht|$Po+lUTIm}7Hr?8!TEn}wskMPNkC`9XiZ|T~XQ^Cu z_j=q?lg{6=U8{p1{8I{&ny_=(qEnL(ngm%${&8Zis(iG+L;lNy;2OSFnIQr<<-||3 z&Gi20Dy%NiFs-55Od zaLhSl#Uj{6U2h3H>L~I>i1VvkX zyzoJ|-bJzUd9lUE?K{(^^0yqnxU;uqUX0?Yx7yj40!)~1_Zpvl?N^wgl)R|@Wc#~c z9L2x;?0yC;nZ)=}Kx$bS%?8PHuP{WO!Qz)&~vB?O}oLOe}B$9m4&lSS@zwW znp+XaD4K50yPrGc((=Gl%8gAya|Ud_idTH2hu&&aa2 z8lK#+Pol?jLG`__RtMRh`DpYUT)a)L%V#f$VfIMlhD6u|%Pq<5k@GrrU6yZ2=+-p0 zoyNHQNzA5Od)RI2{8TeOu|zYPA6ay=`FP>giuEt$F7Um+C+;ks(Jm|6aaw6X?_2E& z4<$@C&TaWR<&9{}olFhs4fDJ*W<2|8DI=n5`sB#3B#UJS)-0bBa#r!%#%1qjAM*EN zR-W_5S*~$joubdi zRfdO|_G&FmY|p5O6Zg8hOk0g5Bdj#ZaJKf*O`UJFmLB}<{Mz($XN8~i_oL^|*?gR{ zZPHP{p9;5LrcZqoqVibJKcg*qBLCMlwmgT;AC#>8Q7`X$WrM}JZ5vhM)p?z-_<#Dk zFmTfPDg8AMo@~$C-~IU4hZ?a{woV!W?-p7*@3b^AnzH!tccxQ&jZJt@^)CFWFl#b< znw*92T8$-)?XNg_?R{5;pE~=p%k9;{g?*D_tW>z?6qg?hHVJxvY3|24CL-&86r}%u zy3fYRJv!tr^B33574Ho1+&trXc+;YPcF!hcDwxR}mnr7STxR4PzpnDx$#$pXO*2o; zZF?Cf^uGJxhTk5pUo_vgET5m)arx?mD0f>U`Rc`Tfsfd(ROto3<7$+esng~1L12Y? z){}KMO0(vK+A*@FFJI6h@GYC+`R^YajTMfBTgyv5G|_o3tvBiC1I1*u(=#5e?%}++ z;Jov}mK9lzyr-tly27Z#pu5=4>OqCp-OPw6(PyjLYA+g0VcR{+KQekj$CIpeS>bxj zMjPTjvkGnwozk{Q`-A3Ik*h4qX{+~aZGR|OK1ss=^iNZ+LbIA3OgdHSJfB$v8V0t99<&du#3( zIDc(_(jv{~#a9CwIzQ+n%+{;=!v62%cIMPpwm-pIrLSLyF1xez$FmO#22zJVTgI5E zemL^-T#(_-GPjj8ax6B7rD*J?!und~W$DC?tQT;ihvhGm{UygnfpIIk<63q!b z%4t3!xJ|&{BfhkCM@PxgzJmfylBI6U5pB7p*z=r; zlxXIrFu^+s>pg;hEQk$|+T^m>{YA*!Z7$aHEZ@~y&nvXcR+iYC8obss4+U+SynV{UP{aIp?)UGM_kCZQ^z3BhOskvA&&`%uA9T=F-X6K3CfvT(K7=Zyl(-qzGzG~Kcz)%Vl;I|sblE3>ZaOnScE zHt5-Q+X;PlzTKF8%eVVj?*3aBXG~N(pI80HH#Vv4Hgn{~Wt(~K?|b{O(Eq2#HUF10 zvkY79GoOh3JLK;3P`l=x$Ui5gWgo<)3|seiC@=dUZkNPW|4@0^2hpwbME*G`FZ(ck z-w~00I-e&WiuX7o^3Un>T#epZ^_40@41;Z zH>=X5=4a|nSH4zm`|v+|RQiFx?kX=6Hb1)Z(DMJn?d}Kl41-NIZ+zpuSjjiz*9;RL zKZcBTY{x?$GG7a^eYcxizH6I;(We!+PPNYsTDE$NM$W^-?^KVRex$kS*zzUZPR~}n zYF%xAG?%5lvitGdGiE&Z8GrqpP#V7Cbnqs(t8Er5@AJ+zd82TnibLw7Ny);*jUiIM zA1+BW6ljR^&seef5Odxw@&2WkEat{OSr{yO$YO46hK4-H9`1#SX@U#&r6;YbUb9C3 zm0ZrL$2Elu7dFV>@zDFp7cze{=*i#b5lO;vEIV#irMa1PrQcltBR z$TivZCzebzoO)dHW4_+Gl{E?X4(hx3>f1ctUiLltPgf6nWtmjml?m>tvz3fluKZO9 z+Qut?*1}=RaY>&783nCc)pF6vB{L72W%0e5Q*~n(11n1@&%ykP)xTNz>y(ey>J}Hg zc3TwwO!z=P!@s-FyBnilbYC)fFzev?<9DBFzA0Kztr51FcTKiNOI!R`wwbGM9-9#R zl~0sOZ27NrmvxedUi9+se)U!%;pTPm-LK{@(04k#zUsI4_Gb)<{wII;oj%0FV0!M_ znm)dUS>MeCJKIMUbQ^ZtY29gd^gZe z?ibhZw$o2OUJy?*PHk@P+?BPve8-2My%!oTbF{RjGcC~H#4hI*QhevSp?+X%Y4F$5 zo=%4LgRR%Ze~5DY+LUG+v1nHO85TSHr0;L1but$m5nY^jn=?PJZPB4MEP1NooB8=) zYCk;d`+nO#otsR;*KX~s&0ZchI#!@pzRmiLo< z=5V;p*qoKXd7(o!=Y@v)g;gn65329UfBA*iQuIRCg$r9*Blyf1AMu}f8NN#HVBLjl z@yE2zOxPy4ML+3%j^FQfQmzN%YORH&R?MLHjnwrj>K(Q)gkRiWa5n_iT(*sZMd|b%q)1Y@z%n{MokYkIPET8vm(y$@r={Y^fOi(DWxQ@ zQR_38_-I+m5#2Ji^ShFnQJGWlWpd4SH@H%-_VWx#HMk;{z_)S0%qDNc@%RGm6fTRINNaBfVy6j{?KO zsL8uLnj2>vf6qNZ_FA-r-227LsvRWuN+&L1y(SSSXtw; z!iNv1Ca&5nen`M|`-Z@YmUk5wDj6Sm(bUhH@u}e_L&Y8Qs4xpYBloN8A2i3T6xe?# z<5R_6PeJx?D&9W=c62sMz3!eC=(vEbV@t-Tgwmb=r>AfJy&<^Ll4FDa^@V$E z*zUF`T83>q(k^@IwC|n2!lv>%$D8I}T3MTTX<>+{{E-Ov30mvRdahJWSk>e2r|c1W zjD4|zQ!&?JpTN0^-G`JMBTQNvay7#q9qIV#5K|fWMMRTxS=pN4n-&79Ah*lETZymp<5tHXQvn?}pO(hnw7%m`v&Sf4pd}68|%$TdKQS+n#Q^vm;~s zg$*{M&!%58YrFFMVJqJ?@u!pjR2&PLXqjx&9==jsYT9z6$42vv&Z+jCt~f1mX-f7= zW%jLKT$Ojta_7r>=*%qC^Qbz(_~(Vw{9?)si(lP)v_|em$F_%mg#Y;5&D`>E{sVUz z`RDT=qm}?Ht9@VhQ z=cnks7X8zECUqurT7OSyY5l4ElxW_LmfCB-y)^pY2}f-Z-L%VZV+eOmN#O584Qnjc zujSseY<9Gi*S7SU!lO$7-dzp)6!KuTj&7~W`6{3CUmh>D?!Ht`onpKo#^`0q z1n!1ATk@N9Y;?-Mc}S+P+->2n-@f!f+Qd+1Z?*neCXruL-adW1Ph-UyhIfg|9J@bW z*eCO>WE0l}p?%J?UfmPt&HJFFx-gNEJ#NVtPtz-6TLSG&ZnvLziCS${aAOL?nfFSU zErcI$-1UB`&Th8fLThA8{rz^@hOYXo`9rgD(JX(hw~4)XYclS1KNAkV@FU@jTddnM zm5SQ3bmvu~{D!wrGQT#Ap0Mmf(%*>P4jV395D`v0dTZ%-&*C#(rE|Vt*l>e?bJAP6 z4^GaH#r_{LbCjw+kiXb3|I!{W^Qb-U2ajB5e;e@VW%j=;{q}0{cfSMI^7fl&NMBMk znq9urW3qRuN#*A|cMh37al8F(zMK`q%xdrK+F4TT_j*ohD&T*%LMWU2i?+{_mE{e- z>Qggrf4l6=?#q~C`NCxuizY|8aKF|@vjvMc?==0&x8dayq1@YbRWBB3Ht4d?^_Z-1 z^qB01OOdLVuDvKT^%O3Gd z`Dp=~5v&jSE*|w%y_UulT%7$%HsP|T>aA^mzg%z(W{BpqnyBX3HP>3KdtS(*To3mv zcVIG-c-8JiD#s5R?N=rkIzgFg+Z@JJwrtmI zY?I%ycjja>)pa&oBOcAM;JBb;DEQBHOGUcy5zE4eJ&S!i*lX_2-}&=jONI1(*@ud3 zee3?75dEHDCiz0xpS%L_FXy1eXr;Bac*AiyLyjnU6(31JpNnjTB)0S zmtijZt9m9KgDi_bhL-i>AAO}}t?+-T_h`1smlqs|Jd>j*MSdxb5OI)YpUj@WMDYLN zd0~y~V($7*5L0(vBEiXPf70u*v&<%o2w{icN$m1s%S@uLeogvVmg6s^TYFK8)nxuR zm5GOrH@|zk>zxvR=d%rJ3-(8DxNxC)p~UfM=1*Ia_uY@YByQX6^m$>;4P(8Rr&{=R z{g1tRn^Qh>rcFlD+eqJ=JR9>8Qm!vM^X0(mSsy1lwED08`&n_5#gBakE9#4KxqYQr zyB;MSU%V!8fks~f6E`z^m`UfxZ4Ezork(m;b|-0SuK}yUkGef7pAQ~?(sk3kyef-L zK`}^bmDP{sw=VN+ULXH;ugcQN(KUa~l=M<+R{UIiVg0T@Lce^aB!B6eyh&Rj{jT!T z24VF-KXO<1&NF$lL_?vkNSd?Sh*cqab#L6RB>!bQKJu!xp8tGyODq3wi7NYwaM#s2Gb4UsU;CeFS5L^jUy(e0((wz+ zZRQ<`{FSuxPN3rc2^;ipww>M{Wazp!$=&du(mC$^&rjc8cFW+)hAaJtGHynE{Mr|O ztV{lVRqC12e*9b)eymHsc#l!onsv3QYa(KQpNe4qVVkw)PxbbhA0NKwZ|ctc z?k{GyM&}xj#EKr{GxjB^rp?Dv^6DQ3ZF`aNXvZ4G*K)c&{0kG;|Fk^(qhN;2vFusg z^547`O6$kynS{u>7`C+4w=M3;*U~r9nLYcMF1Kr%i(%N@t{JZ?+GETNIqZ(BPo28% zg2}husa!FSLTmeOJv!8%z9XjF>ZNJFy53VqzaN*_7CvOyv1y-1Sk{YecMnf^>UplO zT5*%%w|f0p&1{}Y+CBL`p31&tv5Hk z-8sp1-r?qkDg5?Z_S{!m|A^bCqd|D;b637w&vGU1H7v?~W#8fQbo%F0vAr=zmv7k6 zG|}(n>F1wU)rWuO%boRXzN^`ywMv@|kIcJS%=NC$A$H-%@64|@t$n1fSyg%9Q2WC9 z7V`>I&9*5>AJ3h-S0hbA{@;(H2d}#q%Ad}ex8JI4!q0e(Uk{D;cl~nOnlE+X!>qUa zo;^0ZILq#l#p?RE7V=U@3;pc2#2wDR`@uZqeZH~JtjoI36O)bRN4)IT zb8+)p3wAu7&DCwQ=E>bVE6P{-@*A#7tf`3n^Vi{(le2Tygk}mSJZcz62jyXWXXqwE3RZ!Z@-uR|sc1U8d!6zNh zr@;nNCZVBwcTMjW_@=mzBh#Wouv9PXHMjSQ-^IB_@fRNnzu8^9u+H?B`YF-cv=aX* z+P{4EY$@xQflEdOZ>h}1O*S@$cP@TfvhZMaQTY}fe*R~3KOL0TGc3=TUlmZ1 zB4M}xx13oG>u<$ zjJB#*O?>LIb^gV5hn*+Jull>DT(n(jf6OoLyV|^`HIL4led&qjkIKfZ?TKmmcUtN` zmb~pgWccTwPMk#S&wv;0@qvfC=bZ|%kp9iL{6tUxQ;|c@RlgM9HrT`X_tvc+5}$(G zemd55wc4KO=kM?Ob1!7C^8RH%)f?|;eHQu2y=gXKEw*?(l_Q=L~l|A*jTRSoyOlo{=h zrrI6wFPzE$Av#m#!Sa`;4fD-x%pn|JfA+dN;z19Vf85bH->k;`LvYv;=076BH4XDO zU6OyW{L5E{{Zso{{!Cd})o}mTbe2C;K2|o&Pm1IJ5d7>BV_p86Jk7nYA6b7mJUz}X zR$aums#0mk1&tl~mo}K3Z2mo|5@EDcjnB5bWfl9A8wN+xfwqg&MJ+(@chiC z_fL;R#h1&TJk_2y(d*6ABig*HPaG49j}M*sQMou``^Ha+@tb5!m9%+TIm>%H7KLwF zd42JkniuDIDz;rM)6mmOnzg@KzR>SuV8e{$u%9*yT*WN=nj+sV6q9<)V|1PM?!EVuu7nzL3BQ|kWop|a!TIlk zFJ~sV7hNvn^x7EuXI|U;<8wFbpITnqvT51j7kU9nQ%-yNtH% zOL)7%^uz>WkczdZNFz?O)rxO@7Kbl5eN+$1mre(qU`4{>9fROR{S3s;)d` zVXwL}Dqik<`=68BbqbWf@bA%1mJnQ5c0)<;*>086)WhdIUe#Vx>gMqo*_ ztC`VT*!g$%>st(AI;Zc5D_Cn9ba--{&7ZQ&!0u12;&t!nsTVyHB;GFe>{_T6{@Z=B z$m7h4#q-|;*cmC|B1{o#GF_+M$D^+R2HNv=)2|Tm2l|? zmnM~fxgbFUE`6uO;#81~g1%!>W@3)6Q+`g0ZwQyZYhH4GN@iZVg1%=;YFp%F-k%gzquCeX}2SPh6_ zW1|nU73LI>5g-GLG7C!bixdn&E)K%&BDl+)JzYRrK}H#I={tM6gyg$>y7(p*AlwQT z4z4UNNzL`lOUqZVvC$7oP0uVYDXLV^a7@WhO4a1j_b*CGEdqH_1Lj3dQ1}!SU2a1_tI9aCNb z9NZuVN~|E_gA1Hs+%j`gjT8(`z)=bo@k`82MNcelIf?1T3T9mTj-cdGqF`xYs%K88k?KxnV6YaC>R+S8|#@_8k#5= z8km^tS(qA{q4*QzcBnsnQuESFG7J@r&CCtWxb)pJb4pT+6!hJ45=&BDQbDPgOCO>} z!O+~)z!+u0$=kh^HB($KS;XBv|M|>RPa{vYnJ16hOimFx;ppVS;jSX=u}Gq^VAg_( zJSSUto+z<*>}J}eA>i_nL4QhxV$-P!3T@txYxOH0ELyNhYxRmXJ07tnK8wEl*@w@i z{Cn;F`}X_!jjsh?y&HP>YUtIgca5D{m>3v0@UWc7$SE?jli6q^ldwYiKtS~S6~+5r zev)C$=wj@c8}od}wm^A-BB_QMfnMu!SCkxRdZEtHwTxLpe&x#So!jzmPOU%sZDK^; z`jv%NPVqdIyp@4$LK`B!9$U+(JmFruK~mn^N3YH${Vi;W+|MxM(JJZZ*KAe(T;?!X z!sf7ROHpF?pYw;y_pwfBf4tf~?3$(2^LKM9v>kpVHf)G)cZ18n4%YYR|rcTP2+IY2s-XG^NZFQrT9ZvZ3=FDmf_~3grmbGT>E|PmOcG(N%jU?Qg*! z%QM0Id8a>!XxPAbu(4pdiSV`L+p((x4cf0hjxbM_|LAeQ`Rn)9+YaS=-0yvx+A}>_ z&0bjV__vTH?1EI(g$eOHO=e1>2*S zUHj7f>d&S8h`A{fZY=e7Gt29I=NVla*dA(a;h%io!IFF0btTz`s!YMxf=w7QqeQw~Dgty84^> zkwJ?_%XY!MU6x7ImdR?(6>CA>3DIX6eIaf_R{>o(8?tH!)-8F&Nd}L3> zw+eI`M0{n5S$iPiN$R(mhm)qQ6|cXqvn?X#w_*RwpE~bOUeB4Qv+UPH>!o=!4wXE3 zoRss^FuMH4`=qwkf_0bo9#V`ro;2;~Y{XlGf&%LV`b&FU z?Z4e#Bre}jz483aRkscN_BVPyayRg-;8yeO7QEm0`F3xqh5Sc>XB$Ljta;g{_~P_Q zjq3B2Kl0dKIj6|=m9E~wn_I~7?s=K5%i|-?E|a!Z`g=IUP2YA=bw6Q~ z{@(s~``3TY3;auR1A4z)zofWc{CTl;wD7BZ54NT^$0P1HZs1Xjob2@O+8OoOSCR<< z5AU^IP`H)2C8b?vqx7dE1sfL6&55vgF4a80`}KzZ;U~>Ei2W^{Y$fxkRd|MUq0TSH zcl;b{>W>KjK6=ecV;sduLz$ymvjsdarchJ?E>kx7L_VOV)h)kg_NL^RDx?xoiAp zv6d&^S=zn&p)9m* z>(w|vU+x!JdMxJ9@mq}HS#R#VQ@X!?|M9OMd5_)TiBOf?yZ_UV`a`cp&pnX(#dcks zxheOdfMzZ8_lHqYN8dg^oPFb$;J(HJF=dPWFaOwlF9asdhaszsw3}f7kgWmLz}R*s))le`*eW)7<@CAzJ@cp26z37IvQuWPkkRKQ7w0 zcwhMetuv}WXXQUgoOrfR%pAv{7`nuY6ru8 zk-uL*dc5P?yXKmQh8H@oR@D8Fns&J)LeuBa4(rb9wuYhzvEalF1wvh|@jw1;4!Y#X z_q_W+?H_Fo!P5czq*oo`TOqW^W47S6WSd2EuCQ)i;$@Q_+x%CvK*k7S6(b26q`Se%jb*trPoa6s`xzhI5H?|)3W%`%?eofu=?bXT`6^eg<*hKEv zI9}PeVe&5V_quVj`Ky;**t&st;oQ2G=O6MSoXj31rYeg}e9R#&S|OzOxPtwVh=$Pg z385US9E*0a$T`eu`;pz^%pooKP4Q3gnKtQ-@1F~nZ>nZw{-yIw@5qgR8i(py{Qu`5bkMajD)63~OuxNfw(XO|FRr)i*>rERU!RaHQ7`?Fi_L3i)9ufUG2gr9Z>SSpZ-2gCLR-0I zWrF>FxtC9Q{yhEs_t*PVH{O5y_n@EIeueQzohF64rav6drx)7q=xppd8lj~hfBr!H zJK68w6Rc7e&g~Ll?%(+NloswmoXYDs*Jk;Cm>5({`68Lyp9 zZP>=lc4Bvvqy6q_M!O%)@Dcm-@8j{GKmR3Gb?PhL7q3X0#~1kg@x32mpN=-nnE$Bz z`#q!6s>jbieiTShN8|k=9M9iz;KQEnUv)Znw z{ET^`?tz`54C(XPU(D>Sz{O&XNj<2eSg9+EAgZ|z2}mqd`&Q^y60XYy)d?? zEqiKls>(F(+D*^RwEy0$`Q-DCXI|O4z7Hqo$95LTU#fRKpke)6>5;YI|BgR-Q?#A; zu)Ln8eC|*Yr<35c`v*TouylRNXc6XEkKFfY|IPhoBkP`}J$GmQ{P~+=-8_|3>0L6LHyc~;-vlC*?sfcdd&%(X zGEex1YP-wa&hY^XZTcJ+{Ei9*?Ftgus(d4T%bVl?iO%0Qf{L_v&b9Hl^F-t0C8J+j zwR7uq1RMpnIPKZsk|CTYCMWVw>($YdkFGw-`N;QC?_=G^#ErpswCgnHO(^r$_loyk z9$FQ;<6UprnEUM;&Cy~=r2Jo{aX7vYu>eAC%a78WhC z2{|{R{q*9eF_mj#6wFU}KXrd{{E7OSNv3C{5_>nRMYTsQk6yoV-S)iB#f7VOJlJtz z$A=v&EWGbb*fC|t=|?h!=PIP~JGS3C;Feo8`(WCZnYo7BSfW?yo}QMydaLf{wHwxM zx|_56@V5nTIiuTi`*Y{NRlPlX8}l~)-2WbDe7C6gd9U$U`3@hx{&I-X|Rdb-Ne$Lyq0l%AySbj|)C9dmx&?*>tMuNhAJZjMWhy`H^O zU`tHO9fhMsqJ^P%iax!MShwVQ(d(SI+t%)#R?VziE}?6_>w3|-BbD)jd6q?9Hr^MC zPNusrdio~)_(6gW({G=!+Z4f>{pW}8f<(1c-~@L_Ue0YEU$ge zJI{FRX0iYIySDycY&zasS55o2=0)Va$FC&$K2%yBNO!ND|4M$hVB+8V>#epuy0O0g zeo>C#w0VvZ?H9`4{Mx_0&g%W;qsPD1%&U4c&3}8%u7{I(=i7WaDb?U_W_;7%cfQUK z7UMtalJjraOaAjbbaBz*R`=rmJ}aw7FFq`mbZ?jGwU(;AQZYs5=ktmAeKn;|eoQ<+ zRzps2BdVlf0=)b#ud46qv<-4PPZb0wUYhDT=o3?tp@%)+B^zSe`^Qq+@&Mn!~ zqV=PBx`AiJH*4MP3J>ln`3WuW676{YCI4u(!h^rPhu`l#BE^@rmh`c`n?YtvsSb+3`R zKww_e&JX@Oc<;5BKeX++D*Sg7;a`^H@@sq(H7W>KWCl;p|SN7y6_D^1$B3;>J<2mopoTqBm z9_%k$;vWcPDrcYY`($#{b)I6Z$^@2)cT@N|)!2lboE}R2ncE_$eAD~e!~!A5Jhk5^ zH$Snwq zN$DrPpTwS>**~dbl0%NlL>UG4lh#jSKgm_ff199UG=oR+A~ns``oYCo(??|D5^9sZKTBgUiyp>V)MZ#g#ky8C9nQHFkw8(pt4l zYSmJ!Np8$5=CNH;kn-U*X=%G;kfq|BY0<0jQl)v;ikWVgG^Bi!GjwEKbVHVz2W$5# zANEk|RX?1`wDqFPEEo36iY7CcT?j4Nnli!ZMfoA#L-8%rty2$WC9W-4UC_QEZio90 z`SNq_t;bt)4^2K)me~Gb%ZBJ1`ZoG|boS`&>COHs`*6aCg&#J4D1W%4z{x_hM!m*= z|MLHf>@QsY;{Hqc@63Nr^-I%VMBY+<(Zb-^znlA@r>&BmR-Tx=@b-@W?&zbrkAxph z-myAI{O%*Zjm+iG=03XoNbjTMo%FKq>Vo`{?td;zz`fPCqjJX!w!v zqt}mIKWcu&{OI;0+mG(=Vn5RS$okRlN7avhKT`b2{4sxrcdbU9ikx77mwfm9j`>~3 zkL>TB-#NcKzT^1O`i}Un_)hgB>yPFi$v-;zk^Ry4M}i-Hf3*LI{E^F#`kxekYW*?! z$L1g1KUP<`*>GDgXx=tar$VaiV0VS}tCqPRgml`U-+Uf+C?#=KfRc}38i#SKc56`L zECbm(mH10}-z>TBFsm1)+Hq~4Y_ntPlgX`5%dm{IQ?Fo+2zNd7r%lcDpjsHIX zJ00n>*gvEcZ?EXD*t^3sM_m5M++T@`(GrS(6gK4V4Q2SJ-^DraMB}{aWg7woRHsHQ zzRLQ7xw(l~abe=I4;+&%jkK)P$|ksOJ9bq4hR|d-eosG7U+G!%40Vbt!oGFPnYh@k z|K#43oO!nOcYZV-TI8)iyP)!;sIa=P-b|y?TVEuN*X!ucF1>p7>Gd<~3J%OUrIr5m zj@0!%Kev{Ct8ZVsv*+8i=<9dO ze#@4xyI*kq?)N*@_mBOSzj5W?1808wPcNPw-)6I~boPfMll^nc-@kZq=jWGo>AT9_ z-`t)5fAjfL`~Qc!_4occwR-*Df5&F$@BMq7nf>A1^83~QFR<5J|9R-{-dFeMR1!ZGYaoSEc;Si$LY#O`F@&`A&Pt&zd%Cl1)lM&>~$$yK`F6Y)6l|xVz5s z6?}MX!vz87f`A2f4$H5u2%V%6z_haUqpRp9>y3wW)sDrNMM`NOx888*;`&90bdy6b zI>>F0Ie7C)-m%_BUPP9Io#dOnG>xonO#Y3~U9qM9wljq)$SM#T0xt^|Q*$bu#JPxOF zyYlZi`XqCzuUJ{|g^z2NL&zyv(fEKDeLeaKQaf0q7cM>(YqWIB28CN$y58QUlRr-V zSbRso{+#TbLy-wK^2gRny$}1Gm?A!#cg@7|C1u<1-brn+6xIB->W9|xt4s3^1|B=c z;d!u-`_8i0`wYH|Zv1@V<5KP4+ugWxThpeW>B`O3PA&gF<oFQ>@#-iru(rDpzUg3)Zw*;DiD&i!x-3pVsG%YJpRyoVHg)mki+{qji$BcLvp#dMsru%Wp!mo( z%ZsuJ{{EW9m$|N8dYjKV|6tU#pV3uEwI3NyD*xmizax6`joc65anG!^sJ&NTbkRzuiy4jElLF%Io<761v}$|C z;r}5vD^}lOVpCnOP&Dy&K;g{yk;^tm)rT+hUi(u0%2V5;ODtXq2X86q_mhl}GhSfh zU|l=A;a6AZmWp3%wwc{+*rw#;keloJu_!Rwb@t-ZyL0^Z2EoD>V4BASYH3rmp!N$ z^UhyZGSY6*<%DxOC(XI8GYQUdvOUtws-N@KfTK|LQN)qr4-B#=zHTXJlQx?Eu%hzB zJoT2t0y*c7PcBn?#H7fz$TB+HeM0`6Ykdb#&irYWxF_M6&Os-I6vb;Culjtt_k?d) zysRuSbh%l>%k8TU3(VUo&(EJ_pmg2niBV+JcWx^W_XOja_g3U@e9s7^OSuwKf>8DfB9X3_8*o< z(vIBS!n$5l?3c;eHp7l`tsknp`j48_w!Leb*kNtmG=u9MpQ-b0>kuB>tBm`CJa?ta z*LLMKYoyd}Qa0qu-d>`n7SZ%QslLU|E7~eaci~T~UkaM4%O|dsVDseZ%4wS=BchiO+?~ra;&+ zhx$P5;!d6^CmcY;10hL`iH#uQpsG=T60@q&j+P!FBd-%0ip89gswq<@1Ufx~@tM6O z0uD6@vFRl`aDv$fRC(^$ujTN0B;h6F+?lLV*y9bMc~tufL3|$7zKgEbEzLjm7oTtx zvFuP{-wCAypD2if_<>Ir7=0DX;n z%)@u}@I0*0tU3Kr$l}b$uN#^l+aFHr`_FRPavxujMI1|~Ome-<_6ApHD;>*&KLuwn zsVSdv%W$~SvO)HtC5!2DK82dD2TN~$voMUXjojzGs9WpomMuHD>f|clRc}{4 z54lu!)lhWm=f#-{=NHe2dJy(6rtAEBfdb)!>2k|IWr+37Zml$vz8`6FTCOPQpWaLv z54H2Eli!|vaxG=j`lZu_CZ=0S7JXQks<^bQ_EE&?#FjZnmTa6cGwNCNxjzoY0owyy3&HlqdRIk88%hbM|oBpmZXKqa#6ncWYb5 zo86fTH}Cu`QpmVz;n^IRm+v0LwrE}7l9L_tUV6+*K6!mo?1UqIj!!bb+%Y?H+gc|+ z;qzp>FD~t=`ipdBr|GynmGyKMjJ-PX%QjW}31=9(Pq-}Cu8=x6`{7F2X){llEj?_+ z66oa~r1aSDSYyr&tClH;#5;1kLYnfI{NOt+;Zi6&EwCq=+4Iat7uL)eTh7MBg~Fo0 z^2{|wJ8cgXT`FpS8ZGjljko*Z$Ax!Pm8=q8D;dAFI=ycT`vlATmnN=H{(Y3t4xZg+fT~PJ%UBy9NcVA6f?Nu&N zTAJWv{!@$*1qk?D=qPn&|HQgzfk5n#^Dm6MM7Fk!R}EcVg0+1rI~Q z^v%SkDBc#I!@Jm$N=BkwafHaGb1?i@P@-L^Loq0c1%Aae!{wKZ$;xX@yO2` zel*E`NP76?lC1u%xva8>-yG(8EX$fYw{qvwpUlUi?plXszGJmIojP~wrj4II>0g+0 ze%9w35kGU*)R=|Mv+eH>eUVfcAGFJv`SGa_Os5Ogc`qdLgu6taKISSRb$nC9J=O;u> znYK~EvlzSXp1j<3SNX%gz0PZoE!5&VwIOfCem{YVyL@Z6Z8&|qw^wroOJ?F>L#@^& z3s~P;?I~TExqDTt@6FJKOO3V6HYeJ@d*ChjLC#&gbjj!6MzL2U)04kXpLXs`j{LFT znulcruldL9*3$*G0E--)6qG)LL_QtE9H>?3-b?Chm)Dm^LMC zzv6@(P3PprQ8|0qt&@^|9(dd1;i>7JsdguBzp2?r>AlxCob_6NcF%9I{mwOC&pt^0 zX1Z+4uhuA0!FxAOxlWk!&4H)=xPqI{5pI!+{U4f-KmFUZ)zD}EF%#L>%ulZ}`X9J| zB0lgzvBwCkTBy&!9d>pwD^c_qYoOyf_YU-wqLC7QkFTLJ+e7;4- zV$FN}d*(YSO*we@1Iw!^PP>ktKEU<)q1%qxE?nysw#vyd@?X4Ml(^@MOrHs3nv<`w z!3X|}3Tj=>M-v~V6i$2LdOpQ#y4U3;x|hsevc8<66&|`YE2MU5+`71hm9Jwy?ew>K zu=3*$+qn|ErFxdUz97VZz**wE^+opl1+M?Qq~x0~zi8D+QuSQ%T_eb)>!eGO>dsjf zCyO>sy5sjw?fb;`6TueDbC^RHztc3C@3)xm)e`L&Uc08fJMvxbm(||}xn=5s#ghf} zmv9_z;rvrQr)rKf_kY!+xj*h{$q3vN-Y2q8$ZyegpGOKCL-+XYaj)^Yqq$ExR`Ke? z--<6SS4LO4znWMT5+b^^D{$%RXDhd@unS&𝔐AtE;cRU$xJW^5pS!JNxusyvSvb znC9jeVOb*6CryqJ><;frm{ZC1D^Y4)m%M`ihXwIX5!SE7{FILxsIA=8o%gMg7p->nzf*mjpni6#@k=+x z6NPs^^(F6#=PsZ0bBW}(iIY=IcFub;Vd)3EUV&v3MJK$j?dWpxR+|vqAa#H@Ni^p3 z@%t+$2NsCOrxX za?=9Mbu%7IsQ%t~#w+}>*Y!lblSe{hnKm!rT6cU+`Rwa&@4ookUA}QU<2{#~#wt_i zD8AjHv@1YKJnr!KwI38Cn_qvpWIylu`@J@y7kJ}^jW$EYmXi0C6(5-zUi@p>Ejg?5S_hr zbD(2D=bPo8-xQ|K@>=BKu-JU&q6OVnZy3c6?N^C*W4Yui{3fb(&F6)1X;%8DtpNrJz@o>scomZ`=s#?C(#3c3U zSN*jzS!o`*C(g$HyySh@$YY~*Le$3U3sWMO?soi8Wo5D`um5`UvX?g(3b}6zmy8T} z_q*(RmgooHHHl3Mw$CfeZtjfjwq1Y0FeT)Z^!f>%^ZYXX{iUa#mvy$k%c`)>!8?25 z$$85sx=c+8K0S-)Z1F6A)1+N#bK9~^6gtejR*0FrvbmZf8tN;0!(P0560;FgM#GA! zoW4iBR@jStleHb!R{5-Y5fGg5 z<-vr5&khDQvuEmY@9>(lZmy_-|DVYfb9bs{gQ--`Xi&YwQhQA|&o*`UC9K>H_j>2_UiDmPywY_V>ztw( zH<{jUpRDT>6Xv>3UnpD>`{kM6>q)-*BK2ofO!A%Od#~Wv6Q*~eC#=h@mj3CUvS+g5 znd96+MH{Ok*rrVWR5U4owAds+X*^>)`yv%h{$c~^bh^XePURYpSX57=gQr3NGba8ghX>@cH$W4&r zWAk;nRA!p!y(UtvWo|?pa@b6sN%lB^z^K#dD9sK!y=d5&2 zjd+Js7cXwoR8qRLB4}mk$Bz?5o|`FIYa41B*4bJ*H7(P-Y4v=a=jXWke+%DoACE7N zEZe=o zrgc~2q<0#WO)U$3;=55RwM?G9aIa=L-Al!_2;d+YFi~ zhDhA*y^_Il^VFHvHg0xb0dGD}feel%jqlynx30dkD(|Y})D=-TCH$ZK=&6v(=Y5~Q zbw}dV+A!lO`af3wdGhkp}zTBMkhwavfvkO!8ZQ=|6?(sX%EN|kwf0@n{iDR@;V+{H#IKlXL-|i?ZOonVviIj`)s#!$ zrx&Lm_1Pe6SR=JaRxnbeGBv66$(O9@p8WjF=uM9-CU zJQz9sh){>=kzL94y`ss!%6P-X~7i)_RXBORGvRusefP(|RwaI|-;+$8f$d-zG81tw-Wqjk<)gPuQ2xNnWDz(`U>zO>*H|_g!b( z56)<{hsQb|&Wtcsnr*RH^S#`S>!CU>w{7CtC9WJw?Vn=5ss35*_Wd=kp3Wajiq3Yd z7cTPRUZ)|G$FX2O@80j{HeEemu`6%i$F3Q3JbVKKkFSc%?oQ4rIy2>v;k@<~x2G=gQohdVtI}Sjxt%eYe)P|pu+;8k zrsFNC?&4iX8;>ToUMyer{^F}Ijx79Qfp4}4PP%nMM^;QgfX8ZbsPI*b4jv<(g-%HZ zzu#_{XL86$M&HQneEM0|9dGBJ{(D^IjP$-Wg+ljrl64N8vDkM-E@17dYoYETvttc| zDkY~Z-{8!2OV8v8L&nA>9}E_A`kEX{-#k^NsPZ_s-1(h%!{2UbPxFn4`8Qp(c&@o; z^|`sbE`MJ7Jp8=odH3+Cxl2y=`{mu)l6$@R`)1iRwsW@pk8Onh<;@S0iV=$v3lx6C zRq7g(A16~OzSsMI*F(|I_j%-J_^t>j#-8s@>D;dt3`=JfFSb)X8hshe$?`zts4d;XqPihpm4dngHZ&blF&TsXOZo43<#NyRps zJ<>c9=B|v3-(9>Xz2o2p*D9aw;k(T@XnQR-oOJrd(p5@#pEyP{m9H#w-MgyV_kZKN z4gbWyo^Nb4JZSjv+lw1(3aSG5EI)QCQsy}C$A?o)Bw%+GV~lFKozZ2P0zH!q&(pS)pSioJ%(thdd* zm1(~hB-A7vGTeOT$hr8`%I7!qe;4SMmMghy1hdU_ntJ3&LHm5s$z1lE3mFfzT?q7f z$tUdm&Tnm=j{dXg&p*yHp5B+BxpWS{e2Tj^>qf`s8y_6YOvL{0jhD3dGJBXgsfFkG zk>Gs{;rxku(aacx(QTE%E zge1kqCkm|Y81(gU|7`i=wIxf_$!ozr~MUyiULQ-lHnw)8J9imsF=AM&S*Ui9j zK{abjnWbX(?r9ZUU8Gaf{&jbFpNLA&Gr#QiS=0FMxt{Gp$w@92<+`7?wNDOysA6+z z%JkQ#wS6^DJrrI!e~OU()D67fo~#H|RGT;1R7XA8OHF4b&!UGjVmJK|3N)OQ6TS8R z#=@dC&%G;8sk(kWY`Y_Sp7p+$n~sHUpSQc0uR15J?BJ`}=W4qrA6xKfyR6F1vsoAO zE-Xr`RNQ5=dz*{*^vt`r^4>bWEsr|6Xzc~PtJ{)WRaRW7cq`P@c;lghjPuE&GmEu6 zmx`5!^uF%7U$llZGHHq1%<%g*AEq>@v($B%X;^y)dr1V}D)0SNdn6Ny~;_YP!I2=Wosq`NbV6zdZ+f_I?MXp z{TE#%qj&D}tUf35Fk<`0j00x`7-zppE1jX4Gi{lha#^Q}_vBQK3nDD@4Lr2o{Sqh* zSl}fu;8373NrLgtSBE_}PQKq6Cq7}tapmh1_-tl=b!(V6=g{7o1QrQDDLny}KRr{+ zgDl-6*>~KLddJPO@yzGcLsJ5Uqk=9>xHRkgsrPep7Bcg^|GZy%Ukz_;J8qJHX?uu%R@_DJwX?3S5|vVHdAn>EgRt?$zz}Ko zm5a3&NiWVUnIOuxq~`vy%1Qq1xwE(nu3y>3b9LcWLA}i$H3A{6)cL@&G&%I^D^b4VZ|6lx6~5dOI=iEh zVdI%yul^ak9y-@pms*+~M> zh317xD;R}34?L51cxiO0&wcfY3489Uu>E0Qc5lJIl{Q6>#1o#~e4ujd#|oc#qiBYs zd6!MTo&FoS)VgG*vLVYFn@xsnK3Q_F6qn3B`t`?8R?*b(xT=#gR=!Mn6#JQbQ@zZr z{Z~KRo-<@#e5;|@XW7?p>>-M)Dtsqt9ucW8e{uKN?n&3wv#%agQs%uYy~FxN=#IK%)lQC&?(A4~Wa>igUh|`WRUKzfj##0^wZo>M zs9<7g%p5B{MNLEZ`MGENPWPSn3s_we)Nw6~&*#$dqeqXf7Jl?*<=J1^z1@e!`7(Xu zME5EEy!YYjN$baD$Jq6hx6jHmyFd9)#U|@e(d$oZHy0Ne#WB@LoIY;FuDapVMnCVS z^)ix2Pk+l$dB#3Tu+X)zoc*5U_nNf;@Z=QKK4DbU#nCV^$Vux%!V!zZN)sjvR4aF+Y^(5El63Jw1BZ2) ztl^!M<8{7eZ?2@AKX=k-qxFQ7dD2b%Z_IC`JiN|((M0)FU!=Z8*A$n^#Tg-iGj65y zZis%oS?P`8G#zi2pC(N%Jl7+?YTZyZi38Xzo>+nzCflmyk(TCuTKf zXIRXW{#ofY`?%qzA40p#1ca+jZH?90RQ2J^0ev&CT`AWqrnv+ctbOQr_?^k0udZC$ z^U}=a;|*1=n0Q(4U-CoGg8M^g+&O+@&U@G2-fFn8=fS}r4*WH{wF{=)TOoh+1m9nc zFK4e?Gr2W>S}%3Vpt<#?(vP|7*4*n|B@`+&le)t{$p#(zxI=%Fz1ZP*-jXU>TlQ_z zJD$cf|3T%WRjTr`|6lw)IB&jl+H|v&6Na5ner~#S`F?Fh-;(7%a}6!jV)@M5MEd7e z`rqCceT_%j`kauUy@t)QU!NY$&a1!9`=(#J#x;NPyW=|lw$*m<7vHEhSTAmBO+eCM|3i-T5Y{|Mn;xrS=r>PunnYX;$RnkymBpq&2bc20%t4r{Uo)0#U)gG5E4oHkii%GZ1ut}YAAj9C)F`mU8 z?Z2~c|J{Brshn@!OT#~h?H6yaPKkWy@=fy1%TFdfU8`(A<{$t{& z*;Q=pj-1HNN^(1VJFU#4=G%vyXpX(>tQ9oXUKV|xV;^65cLl>+Uh@eWh9767|N1m# z#*6+*?f#lF+vdqwe7mPS?`p>Utuq9db$0GpoWEuEI-Z%*n;eCELRrBR_X3=M5$B8<4Z9{jFi`Iz+J-@$$Bu71gTAywq%Y_h+8$>&^- z=Wm}sy_Qoq{g`J|?PWQ`riq7^T>KsTd3&6rl5X&4y=*VFiL=wr>`Z&jcI%y?d~x5* zJHJ?89&;9|KP$GL`F^}0`-czeiw%n>Z{+fIRIX+Ex8E%8)W3MoIbsJj_O8AlG-K&{ zkto)*sgVn>ewd`?ywG&QinwJLqB6MeEV^Lzk=4#``nz%r-T3$0%Gwep3$4VfGE zLScv13!nG%U&ct;2>kG)(Q``1Ta|_F_3#Ub+TEED& zh8>=Bp{n(oLqtT2Q}#jwrO1V)R|IQ37BVM9xHLPxPONa*?ZnQt*@<7PM1ZHaqW^Dc zS)jw+CZ9$wU8gCX1+)IjYm_ciX%$mEByu)W^xA5l=^9%&7A{Mi6K1@BhMKm?uR|vD zE=#SsK3hvPie>XlFTZTj|GzTk*Vk@2zu)fG^LY8UpV#+)`+42{_n+5`YyW+{XopMs zbcXEqtMf%>-CWgpnK42>D{B1(B_5--&$i6Fy(Dt1K%QiE5F!e0q-A zhdL|KSzjMN=6`Hvzw*MFXZ6pwOnm(~G-KI${yBWds}@@ZEp(l5*6zQXLEUGKR=2uh zspZO+|4mxc{!A~jnsu~cXXHXdwll?+vZue!Tyw!ke*SvhIrUqejQ_p1oO>YYz#dPK zba{c*%$vvm^f)c9J#*V)?y=vM!3!hp&l^TvKL2l4-M8ysW=~&sf6te{KfZ#{pGDt) zr&{T>xfM8>S}YLS$kVUv(=O%b-@>-SJtJ^OibVfXwIw$%H-6PSr}4P*VL2Qp92?(eGHj0v)gJ*!OE34 z;*Fhzb&jNqUSBA8drrXh)tvmH z!tSKiJ>}Vsyjk(bt4k8Sxu36lm~6OuyN0sKXUVXln_~Ck&Q~YSczNYOX@O>O=960E zUag&BXKPHZ9s04_;@ivnr`#w102zlF`o^$?sQ?1p`(3p2$ZrQ{vCpfBZ4?{_E%eGw*%>nYL~c?_96G*QZ1G#ic)(r{K&veY)bresjV3lMCNu z$?dIMz41rRGqLr7RnlCOC*)hbQ*h;YTzqBcutuJuBD1Wbi^8D{oqW{0GZNGo(`1bdv3wUQ$v|K63`(N|t=6(6# z;)>TvPx9Z&Zr)_x|10PEsnp+}<_AYA>Ue*BniIdSK=bF$$dmHLmHOxI{W+Zzy{zY$ z(Mc!O{PQpG{;Wyf_NOuRPvEu{=E7D}ufBZ~Ywv3lzExxA;;7EQ+&53}e>C&xoWE5s zPfP5VUGLSYFLt`PLhM_nt@}rxy_Y_>+Dti;Zh9eT-<$upSNwQ;;-Aeaqh-zWvt%r? z)+W78Q;q+7?}l^bo9o|CzwEAm%)Pv5(aZfsHP)@o*;h{Jp8lpM9-sgJ-t$dD-CzHH zT2UXf?bqrZOJo0*tyxyLeBG6}cgyMuZNvqza3c;7w#*)`~UOK?7y2k z?Zv+rnklT;Kfk4@d-K2Ss^^a{udXk-u6grzdj0)XSs{-W>!0TDS!h2iFuum;LGk~! z&;LBP*|y(u>g5IbX8Sg$s(#P#7d|r&%v`9<+ znb;lezqT7<)#S4;H@uystjJwdkUr(3C69Mz#q$$RCm71k&tp8wmCm+}@9;F8)^m*y z-|(v|^)%${nQxG=_xguoPvySpeeYfW*iL!>o;kcm^?c3#?fwR<8rGk4-COwZHfyZc zwp1(Mr;@j}xBCU1i{d?{Id851eZN~rqGRqbU;c1&d;W$MN!zu{l777Z;`jW|oAs}j zKi=G$&hcz@$-eJro&`R#_UC!~^v{0TL_vGz9~E=|t1td>`_lRDG7v8Qp;^NF5I_}b@h{vyuQ7>zI|fh&U^1fL|caF6>PEi#~W<*+GpOifZM0CWs(bcRNc!f@nF}{()i~F7Z|*C-v?U_1Y=OYF-K(Bz zrEkv8t6eqc{*$6Br>35GT9B9=>pVGlUFO>2E4yb!KRG4kK6&vbR@Pb3F;mYLdG{`k zJGSj;k@MC!hXSQ$&j~%}m0n!z9K89>iuIFZZ`5a}_Pz<)Rdn^V?%J<@l@nJlWji`I z;&#B%C6ckP4;TjAYdrPib(EOp>*yH2C+l1mZO!}`Fmv{j#D$wrHeSxoEV7DvwV{(U zWZNT=**6o;@?Kq`7QS%T)@@>*>$LgeuQxRBc(poecJz;s+0kOxt$s&0SuUU1`zU8s z@zvtKWoo>_Y4 zhV*QyS5g)E&VM&r-7?==$h$Rkarz%T0m-mdLvxT=*@=S}lm!qG& z^nLNy%KdVU*>n4@+Bs)qc%0SUWcsyI<{J9y6>d4v`RPmNrzfvoEsQZu&)r-c$h(Pm zu1V#umPMS8KQH&6VdFV3MkU@bvF72D%+{iROM|(&)y4c~$*thN;=A54{YUOunZs7c zUOwCty+gk0>%pzp4Yr@W7_oUq|KUUPBeTvWRTesHPhOK^*mY~`weXKW4zD;c;nBq2 zz=ww_+{%vMQCmLw==*@3JV&!+By4z#Z#;57IgRm@!DQVe#}_W;)-&s8&6%nzA$v9B zuif4mkENekS#tCD#95YT9Q!lpn%Enut2qZsZ{$09o~>JI@Tc1MqV}1jkhm8sa!bz~ z+)-kZ9P@R$oYwgj&nND?Vb48h@*|lT5x$)}FGjvxRTlcM?3q#2#(BpzCYOeU?wrwY zST?KTl~?Lj-#goE)aNV@S-bk&QMH!lf_<*MzMp5;EIp&6NTgI{N@OMILV zZ&!Rhv10MR7psmP-K76Sl$%$k<94!B>HQ}W=b|sX%6WD;VcVJ#v#Z+189`s3ojGgD zyLM}+#A@!?bFTYTzR!MXZ1-h}@MpKs_@p;;l~*0TT$t)TzyIbDnXYSB#otX^eMfsi z3incFnd29YW*MHDF-?E@$!7Por#usj+j@OYpEz~mLRPv+a_X@SiII^%&PdKve%^CJ zZmO+TK=|cLmn^JoYcDQxb$+`~ z7R#RdKdxo!B~k05cZ~AWeICi%ntarrk?i2jy=9WaooD_$E7m?ce(ayA>#5_ue_yqP z9;$n&`p~lG-Hrzt>+biIu?FqSc4W@amAz1}sP>Hbt>b*Fz6VGQIHsCq-zK%bH z=F6qs*Xvm3EzG(5YKD1Yr2E=2xzoeE$QhZ1&9xZ~u|S7WV&i zzGtb-mGbw&TLm^+o$r2b6&R>Al__(x<0Q+8Dq0gx$zBQ*lI@WCvZrKv`1;!!N1127 zJ(=o#>yy`wrZuPCxy(fFBnEW7W|3GjQT3$i*15^Y(A?j zQ|AufJGJQ{FMqRMbeMGjGt?`JQq#B$6pT%{4B$Y)%+%D_R3S|PB4%o7U}2&FmQ~1u ziCGvKnxcvs8kiVc7@&$7m|7Z|p{X-4wJSel!dqq@bwz|z73&CLde1_p+f=;{niP0`$JU}#`yhHi(Ep@{{0xEL8)n4p_y zVs2oBZk~y`iHRYay{6_CM(AN+XkdmB_J#%)nD!c(8l(Hy$jBT+uZg*(5xSdAEDTMN z+)`4Mn3R9uo+R02-fhDOGQMn+t!s;>TS E0A}S*2><{9 From 3b3250339a12a5ec90923718ced3240d9728e841 Mon Sep 17 00:00:00 2001 From: Stephen Twigg Date: Tue, 3 Feb 2015 18:15:01 -0800 Subject: [PATCH 284/374] Explicitely convert results of Bits Muxes to UInt Chisel updated to emit SInt result instead of UInt so this commit addresses this change. --- uncore/src/main/scala/cache.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 794e3447..9040594b 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -145,7 +145,7 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { val rst = rst_cnt < UInt(nSets) val waddr = Mux(rst, rst_cnt, io.write.bits.idx) val wdata = Mux(rst, rstVal, io.write.bits.data).toBits - val wmask = Mux(rst, SInt(-1), io.write.bits.way_en) + val wmask = Mux(rst, SInt(-1), io.write.bits.way_en).toUInt when (rst) { rst_cnt := rst_cnt+UInt(1) } val tag_arr = Mem(UInt(width = metabits*nWays), nSets, seqRead = true) From 7b86ea17cfc3dedb5989728847c8bf86493340ad Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 3 Feb 2015 19:37:32 -0800 Subject: [PATCH 285/374] rename L2HellaCache to L2HellaCacheBank --- uncore/src/main/scala/cache.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 9040594b..a033be2f 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -328,7 +328,7 @@ class L2DataArray extends L2HellaCacheModule { io.write.ready := Bool(true) } -class L2HellaCache(bankId: Int, innerId: String, outerId: String) extends +class L2HellaCacheBank(bankId: Int, innerId: String, outerId: String) extends CoherenceAgent(innerId, outerId) with L2HellaCacheParameters { require(isPow2(nSets)) From 0c66e70f147273726f3b823249a7f20a520029af Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 6 Feb 2015 13:20:44 -0800 Subject: [PATCH 286/374] cleanup of conflicts; allocation bugfix --- uncore/src/main/scala/cache.scala | 13 ++++++------- uncore/src/main/scala/coherence.scala | 2 -- uncore/src/main/scala/tilelink.scala | 3 +++ uncore/src/main/scala/uncore.scala | 8 ++++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index a033be2f..859e0bc3 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -397,7 +397,7 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac acquireList.zip(alloc_arb.io.in).zipWithIndex.map { case((acq, arb), i) => arb.valid := acq.ready acq.bits := acquire.bits - acq.valid := acquire.valid && (acquire_idx === UInt(i)) + acq.valid := arb.ready && (acquire_idx === UInt(i)) } val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) acquire.ready := acquireList.map(_.ready).reduce(_||_) && !block_acquires @@ -475,7 +475,6 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str val cacq = io.inner.acquire.bits val crel = io.inner.release.bits val cgnt = io.inner.grant.bits - val c_ack = io.inner.finish.bits val mgnt = io.outer.grant.bits val s_idle :: s_probe :: s_data_read :: s_data_resp :: s_outer_write :: Nil = Enum(UInt(), 5) @@ -498,7 +497,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str val resp_data_done = connectIncomingDataBeatCounter(io.data.resp) io.has_release_match := !crel.payload.isVoluntary() && - co.isCoherenceConflict(xact_addr_block, crel.payload.addr_block) && + crel.payload.conflicts(xact_addr_block) && (state === s_probe) val next_coh_on_rel = co.managerMetadataOnRelease(crel.payload, xact_coh, crel.header.src) @@ -824,12 +823,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } //TODO: Allow hit under miss for stores - io.has_acquire_conflict := (co.isCoherenceConflict(xact.addr_block, cacq.payload.addr_block) || - xact.addr_block(idxMSB,idxLSB) === cacq.payload.addr_block(idxMSB,idxLSB)) && + val in_same_set = xact.addr_block(idxMSB,idxLSB) === + cacq.payload.addr_block(idxMSB,idxLSB) + io.has_acquire_conflict := (xact.conflicts(cacq.payload) || in_same_set) && (state != s_idle) && !collect_cacq_data - io.has_acquire_match := xact.hasMultibeatData() && - (xact.addr_block === cacq.payload.addr_block) && + io.has_acquire_match := xact.conflicts(cacq.payload) && collect_cacq_data io.has_release_match := !crel.payload.isVoluntary() && (xact.addr_block === crel.payload.addr_block) && diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 201e67ab..82f72dea 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -135,8 +135,6 @@ abstract class CoherencePolicy(val dir: DirectoryRepresentation) { def requiresProbes(acq: Acquire, meta: ManagerMetadata): Bool def requiresProbes(cmd: UInt, meta: ManagerMetadata): Bool def requiresProbesOnVoluntaryWriteback(meta: ManagerMetadata): Bool = requiresProbes(M_FLUSH, meta) - - def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool } class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 3fc0e4e7..afbf4dc0 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -49,6 +49,9 @@ trait ClientToClientChannel extends TileLinkChannel // Unused for now // trait HasCacheBlockAddress extends TLBundle { val addr_block = UInt(width = tlBlockAddrBits) + + def conflicts[T <: HasCacheBlockAddress](that: T) = this.addr_block === that.addr_block + def conflicts[T <: HasCacheBlockAddress](addr: UInt) = this.addr_block === addr } trait HasTileLinkBeatId extends TLBundle { diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index cc4755f0..1230771c 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -300,10 +300,10 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val myflag = Mux(probe_self, Bits(0), UIntToOH(cacq.header.src(log2Up(nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) - io.has_acquire_conflict := co.isCoherenceConflict(xact_addr_block, cacq.payload.addr_block) && - (state != s_idle) && - !collect_inner_data - io.has_release_match := co.isCoherenceConflict(xact_addr_block, crel.payload.addr_block) && + io.has_acquire_conflict := xact.conflicts(cacq.payload) && + !collect_inner_data && + (state != s_idle) + io.has_release_match := xact.conflicts(crel.payload) && !crel.payload.isVoluntary() && (state != s_idle) From 0a8722e881056c75ece113243f4544e702895997 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Feb 2015 00:35:18 -0800 Subject: [PATCH 287/374] bugfix for indexing DataArray of of small L2 --- uncore/src/main/scala/cache.scala | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 859e0bc3..4380e255 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -281,9 +281,9 @@ class L2MetadataArray extends L2HellaCacheModule { } class L2DataReadReq extends L2HellaCacheBundle - with HasCacheBlockAddress with HasTileLinkBeatId with HasL2Id { + val addr_idx = UInt(width = idxBits) val way_en = Bits(width = nWays) } @@ -311,8 +311,8 @@ class L2DataArray extends L2HellaCacheModule { val wmask = FillInterleaved(8, io.write.bits.wmask) val reg_raddr = Reg(UInt()) val array = Mem(Bits(width=rowBits), nWays*nSets*refillCycles, seqRead = true) - val waddr = Cat(OHToUInt(io.write.bits.way_en), io.write.bits.addr_block, io.write.bits.addr_beat) - val raddr = Cat(OHToUInt(io.read.bits.way_en), io.read.bits.addr_block, io.read.bits.addr_beat) + val waddr = Cat(OHToUInt(io.write.bits.way_en), io.write.bits.addr_idx, io.write.bits.addr_beat) + val raddr = Cat(OHToUInt(io.read.bits.way_en), io.read.bits.addr_idx, io.read.bits.addr_beat) when (io.write.bits.way_en.orR && io.write.valid) { array.write(waddr, io.write.bits.data, wmask) @@ -524,7 +524,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str io.data.read.valid := Bool(false) io.data.read.bits.id := UInt(trackerId) io.data.read.bits.way_en := xact_way_en - io.data.read.bits.addr_block := xact_addr_block + io.data.read.bits.addr_idx := xact_addr_block(idxMSB,idxLSB) io.data.read.bits.addr_beat := read_data_cnt io.data.write.valid := Bool(false) @@ -658,7 +658,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.data.write.valid := Bool(false) io.data.write.bits.id := UInt(trackerId) io.data.write.bits.way_en := xact_way_en - io.data.write.bits.addr_block := xact_addr_block + io.data.write.bits.addr_idx := xact_addr_block(idxMSB,idxLSB) io.data.write.bits.addr_beat := write_data_cnt io.data.write.bits.wmask := SInt(-1) io.data.write.bits.data := xact_data(write_data_cnt) @@ -881,12 +881,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.data.read.valid := Bool(false) io.data.read.bits.id := UInt(trackerId) io.data.read.bits.way_en := xact_way_en - io.data.read.bits.addr_block := xact_addr_block + io.data.read.bits.addr_idx := xact_addr_block(idxMSB,idxLSB) io.data.read.bits.addr_beat := read_data_cnt io.data.write.valid := Bool(false) io.data.write.bits.id := UInt(trackerId) io.data.write.bits.way_en := xact_way_en - io.data.write.bits.addr_block := xact_addr_block + io.data.write.bits.addr_idx := xact_addr_block(idxMSB,idxLSB) io.data.write.bits.addr_beat := write_data_cnt io.data.write.bits.wmask := SInt(-1) io.data.write.bits.data := xact_data(write_data_cnt) From 1bed6ea498de86d51815b0c8b95f861bbe8f6ad0 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sat, 28 Feb 2015 17:02:13 -0800 Subject: [PATCH 288/374] New metadata-based coherence API --- uncore/src/main/scala/broadcast.scala | 380 ++++++++ uncore/src/main/scala/cache.scala | 1236 ++++++++++++------------- uncore/src/main/scala/coherence.scala | 985 +++++++++----------- uncore/src/main/scala/directory.scala | 43 + uncore/src/main/scala/htif.scala | 7 +- uncore/src/main/scala/memserdes.scala | 314 +++++-- uncore/src/main/scala/metadata.scala | 188 ++++ uncore/src/main/scala/tilelink.scala | 307 +++--- uncore/src/main/scala/uncore.scala | 508 +++------- 9 files changed, 2117 insertions(+), 1851 deletions(-) create mode 100644 uncore/src/main/scala/broadcast.scala create mode 100644 uncore/src/main/scala/directory.scala create mode 100644 uncore/src/main/scala/metadata.scala diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala new file mode 100644 index 00000000..bd8a3f24 --- /dev/null +++ b/uncore/src/main/scala/broadcast.scala @@ -0,0 +1,380 @@ +// See LICENSE for license details. + +package uncore +import Chisel._ + +case object L2StoreDataQueueDepth extends Field[Int] + +trait BroadcastHubParameters extends CoherenceAgentParameters { + val sdqDepth = params(L2StoreDataQueueDepth)*innerDataBeats + val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(sdqDepth)) + val nDataQueueLocations = 3 //Stores, VoluntaryWBs, Releases +} + +class DataQueueLocation extends Bundle with BroadcastHubParameters { + val idx = UInt(width = dqIdxBits) + val loc = UInt(width = log2Ceil(nDataQueueLocations)) +} + +object DataQueueLocation { + def apply(idx: UInt, loc: UInt) = { + val d = new DataQueueLocation + d.idx := idx + d.loc := loc + d + } +} + +class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent + with BroadcastHubParameters { + val internalDataBits = new DataQueueLocation().getWidth + val inStoreQueue :: inVolWBQueue :: inClientReleaseQueue :: Nil = Enum(UInt(), nDataQueueLocations) + + // Create SHRs for outstanding transactions + val trackerList = (0 until nReleaseTransactors).map(id => + Module(new BroadcastVoluntaryReleaseTracker(id, bankId), {case TLDataBits => internalDataBits})) ++ + (nReleaseTransactors until nTransactors).map(id => + Module(new BroadcastAcquireTracker(id, bankId), {case TLDataBits => internalDataBits})) + + // Propagate incoherence flags + trackerList.map(_.io.incoherent := io.incoherent.toBits) + + // Queue to store impending Put data + val acquire = io.inner.acquire + val sdq_val = Reg(init=Bits(0, sdqDepth)) + val sdq_alloc_id = PriorityEncoder(~sdq_val) + val sdq_rdy = !sdq_val.andR + val sdq_enq = acquire.fire() && acquire.bits.payload.hasData() + val sdq = Vec.fill(sdqDepth){ Reg(io.inner.acquire.bits.payload.data) } + when (sdq_enq) { sdq(sdq_alloc_id) := acquire.bits.payload.data } + + // Handle acquire transaction initiation + val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) + val block_acquires = any_acquire_conflict + val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) + for( i <- 0 until trackerList.size ) { + val t = trackerList(i).io.inner + alloc_arb.io.in(i).valid := t.acquire.ready + t.acquire.bits := acquire.bits + t.acquire.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits + t.acquire.valid := alloc_arb.io.in(i).ready + } + acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && sdq_rdy && !block_acquires + alloc_arb.io.out.ready := acquire.valid && sdq_rdy && !block_acquires + + // Queue to store impending Voluntary Release data + val release = io.inner.release + val voluntary = release.bits.payload.isVoluntary() + val vwbdq_enq = release.fire() && voluntary && release.bits.payload.hasData() + val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, innerDataBeats) //TODO Zero width + val vwbdq = Vec.fill(innerDataBeats){ Reg(release.bits.payload.data) } //TODO Assumes nReleaseTransactors == 1 + when(vwbdq_enq) { vwbdq(rel_data_cnt) := release.bits.payload.data } + + // Handle releases, which might be voluntary and might have data + val release_idx = Vec(trackerList.map(_.io.has_release_match)).indexWhere{b: Bool => b} + for( i <- 0 until trackerList.size ) { + val t = trackerList(i).io.inner + t.release.bits := release.bits + t.release.bits.payload.data := (if (i < nReleaseTransactors) + DataQueueLocation(rel_data_cnt, inVolWBQueue) + else DataQueueLocation(UInt(0), inClientReleaseQueue)).toBits + t.release.valid := release.valid && (release_idx === UInt(i)) + } + release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) + + // Wire finished transaction acks + val ack = io.inner.finish + trackerList.map(_.io.inner.finish.valid := ack.valid) + trackerList.map(_.io.inner.finish.bits := ack.bits) + ack.ready := Bool(true) + + // Wire probe requests and grant reply to clients, finish acks from clients + // Note that we bypass the Grant data subbundles + io.inner.grant.bits.payload.data := io.outer.grant.bits.payload.data + io.inner.grant.bits.payload.addr_beat := io.outer.grant.bits.payload.addr_beat + doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) + doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) + doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish)) + + // Create an arbiter for the one memory port + val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), + { case TLId => params(OuterTLId) + case TLDataBits => internalDataBits }) + outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } + // Get the pending data out of the store data queue + val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.payload.data) + val is_in_sdq = outer_data_ptr.loc === inStoreQueue + val free_sdq = io.outer.acquire.fire() && + io.outer.acquire.bits.payload.hasData() && + outer_data_ptr.loc === inStoreQueue + io.outer.acquire.bits.payload.data := MuxLookup(outer_data_ptr.loc, release.bits.payload.data, Array( + inStoreQueue -> sdq(outer_data_ptr.idx), + inVolWBQueue -> vwbdq(outer_data_ptr.idx))) + io.outer <> outer_arb.io.out + + // Update SDQ valid bits + when (io.outer.acquire.valid || sdq_enq) { + sdq_val := sdq_val & ~(UIntToOH(outer_data_ptr.idx) & Fill(sdqDepth, free_sdq)) | + PriorityEncoderOH(~sdq_val(sdqDepth-1,0)) & Fill(sdqDepth, sdq_enq) + } +} + +class BroadcastXactTracker extends XactTracker { + val io = new ManagerXactTrackerIO +} + +class BroadcastVoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends BroadcastXactTracker { + val s_idle :: s_outer :: s_grant :: s_ack :: Nil = Enum(UInt(), 4) + val state = Reg(init=s_idle) + + val xact_src = Reg(io.inner.release.bits.header.src.clone) + val xact = Reg(Bundle(new Release, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) } + val coh = ManagerMetadata.onReset + + val collect_irel_data = Reg(init=Bool(false)) + val irel_data_valid = Reg(init=Bits(0, width = innerDataBeats)) + val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire) + + io.has_acquire_conflict := Bool(false) + io.has_release_match := io.irel().isVoluntary() + io.has_acquire_match := Bool(false) + + io.outer.acquire.valid := Bool(false) + io.outer.grant.ready := Bool(false) + io.outer.finish.valid := Bool(false) + io.inner.acquire.ready := Bool(false) + io.inner.probe.valid := Bool(false) + io.inner.release.ready := Bool(false) + io.inner.grant.valid := Bool(false) + io.inner.finish.ready := Bool(false) + + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := xact_src + io.inner.grant.bits.payload := coh.makeGrant(xact, UInt(trackerId)) + + //TODO: Use io.outer.release instead? + io.outer.acquire.bits.payload := Bundle(PutBlock( + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block, + addr_beat = oacq_data_cnt, + data = data_buffer(oacq_data_cnt)))(outerTLParams) + + when(collect_irel_data) { + io.inner.release.ready := Bool(true) + when(io.inner.release.valid) { + data_buffer(io.irel().addr_beat) := io.irel().data + irel_data_valid(io.irel().addr_beat) := Bool(true) + } + when(irel_data_done) { collect_irel_data := Bool(false) } + } + + switch (state) { + is(s_idle) { + io.inner.release.ready := Bool(true) + when( io.inner.release.valid ) { + xact_src := io.inner.release.bits.header.src + xact := io.irel() + data_buffer(UInt(0)) := io.irel().data + collect_irel_data := io.irel().hasMultibeatData() + irel_data_valid := io.irel().hasData() << io.irel().addr_beat + state := Mux(io.irel().hasData(), s_outer, + Mux(io.irel().requiresAck(), s_ack, s_idle)) + } + } + is(s_outer) { + io.outer.acquire.valid := !collect_irel_data || irel_data_valid(oacq_data_cnt) + when(oacq_data_done) { + state := Mux(xact.requiresAck(), s_grant, s_idle) + } + } + is(s_grant) { // Forward the Grant.voluntaryAck + io.outer.grant.ready := io.inner.grant.ready + io.inner.grant.valid := io.outer.grant.valid + when(io.inner.grant.fire()) { + state := Mux(io.ignt().requiresAck(), s_ack, s_idle) + } + } + is(s_ack) { + // TODO: This state is unnecessary if no client will ever issue the + // pending Acquire that caused this writeback until it receives the + // Grant.voluntaryAck for this writeback + io.inner.finish.ready := Bool(true) + when(io.inner.finish.valid) { state := s_idle } + } + } +} + +class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXactTracker { + val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_mem_resp :: s_ack :: Nil = Enum(UInt(), 7) + val state = Reg(init=s_idle) + + val xact_src = Reg(io.inner.acquire.bits.header.src.clone) + val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val data_buffer = Vec.fill(innerDataBeats){ Reg(io.iacq().data.clone) } + val coh = ManagerMetadata.onReset + + assert(!(state != s_idle && xact.isBuiltInType() && + Vec(Acquire.getType, Acquire.putType, Acquire.putAtomicType).contains(xact.a_type)), + "Broadcast Hub does not support PutAtomics or subblock Gets/Puts") // TODO + + val release_count = Reg(init=UInt(0, width = log2Up(nClients))) + val probe_flags = Reg(init=Bits(0, width = nClients)) + val curr_p_id = PriorityEncoder(probe_flags) + val probe_initial_flags = Bits(width = nClients) + probe_initial_flags := Bits(0) + // issue self-probes for uncached read xacts to facilitate I$ coherence + val probe_self = io.inner.acquire.bits.payload.requiresSelfProbe() + val myflag = Mux(probe_self, Bits(0), + UIntToOH(io.inner.acquire.bits.header.src(log2Up(nClients)-1,0))) + probe_initial_flags := ~(io.incoherent.toBits | myflag) + + val collect_iacq_data = Reg(init=Bool(false)) + val iacq_data_valid = Reg(init=Bits(0, width = innerDataBeats)) + val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) + val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val (ignt_data_cnt, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant) + val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire) + val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) + val pending_ognt_ack = Reg(init=Bool(false)) + val pending_outer_write = xact.hasData() + val pending_outer_read = io.ignt().hasData() + val pending_outer_write_ = io.iacq().hasData() + val pending_outer_read_ = coh.makeGrant(io.iacq(), UInt(trackerId)).hasData() + + io.has_acquire_conflict := xact.conflicts(io.iacq()) && + !collect_iacq_data && + (state != s_idle) + io.has_release_match := xact.conflicts(io.irel()) && + !io.irel().isVoluntary() && + (state != s_idle) + io.has_acquire_match := Bool(false) + + val outer_write_acq = Bundle(PutBlock( + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block, + addr_beat = oacq_data_cnt, + data = data_buffer(oacq_data_cnt)))(outerTLParams) + val outer_write_rel = Bundle(PutBlock( + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block, + addr_beat = io.irel().addr_beat, + data = io.irel().data))(outerTLParams) + val outer_read = Bundle(GetBlock( + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block))(outerTLParams) + + io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.payload := outer_read //default + io.outer.grant.ready := Bool(false) + + io.inner.probe.valid := Bool(false) + io.inner.probe.bits.header.src := UInt(bankId) + io.inner.probe.bits.header.dst := curr_p_id + io.inner.probe.bits.payload := coh.makeProbe(xact) + + io.inner.grant.valid := Bool(false) + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := xact_src + io.inner.grant.bits.payload := coh.makeGrant(xact, UInt(trackerId)) // Data bypassed in parent + + io.inner.acquire.ready := Bool(false) + io.inner.release.ready := Bool(false) + io.inner.finish.ready := Bool(false) + + when(collect_iacq_data) { + io.inner.acquire.ready := Bool(true) + when(io.inner.acquire.valid) { + data_buffer(io.iacq().addr_beat) := io.iacq().data + iacq_data_valid(io.iacq().addr_beat) := Bool(true) + } + when(iacq_data_done) { collect_iacq_data := Bool(false) } + } + + when(pending_ognt_ack) { + io.outer.grant.ready := Bool(true) + when(io.outer.grant.valid) { pending_ognt_ack := Bool(false) } + //TODO add finish queue if this isnt the last level manager + } + + switch (state) { + is(s_idle) { + io.inner.acquire.ready := Bool(true) + when(io.inner.acquire.valid) { + xact_src := io.inner.acquire.bits.header.src + xact := io.iacq() + data_buffer(UInt(0)) := io.iacq().data + collect_iacq_data := io.iacq().hasMultibeatData() + iacq_data_valid := io.iacq().hasData() << io.iacq().addr_beat + probe_flags := probe_initial_flags + release_count := PopCount(probe_initial_flags) + state := Mux(probe_initial_flags.orR, s_probe, + Mux(pending_outer_write_, s_mem_write, + Mux(pending_outer_read_, s_mem_read, s_make_grant))) + } + } + is(s_probe) { + // Generate probes + io.inner.probe.valid := probe_flags.orR + when(io.inner.probe.ready) { + probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) + } + + // Handle releases, which may have data to be written back + io.inner.release.ready := !io.irel().hasData() || io.outer.acquire.ready + when(io.inner.release.valid) { + when(io.irel().hasData()) { + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_write_rel + when(io.outer.acquire.ready) { + when(oacq_data_done) { + pending_ognt_ack := Bool(true) + release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } + } + } + } .otherwise { + release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } + } + } + } + is(s_mem_write) { // Write data to outer memory + io.outer.acquire.valid := !pending_ognt_ack || !collect_iacq_data || iacq_data_valid(oacq_data_cnt) + io.outer.acquire.bits.payload := outer_write_acq + when(oacq_data_done) { + pending_ognt_ack := Bool(true) + state := Mux(pending_outer_read, s_mem_read, s_mem_resp) + } + } + is(s_mem_read) { // Read data from outer memory (possibly what was just written) + io.outer.acquire.valid := !pending_ognt_ack + io.outer.acquire.bits.payload := outer_read + when(io.outer.acquire.fire()) { state := s_mem_resp } + } + is(s_mem_resp) { // Wait to forward grants from outer memory + io.outer.grant.ready := io.inner.grant.ready + io.inner.grant.valid := io.outer.grant.valid + when(ignt_data_done) { + state := Mux(io.ignt().requiresAck(), s_ack, s_idle) + } + } + is(s_make_grant) { // Manufacture a local grant (some kind of permission upgrade) + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { + state := Mux(io.ignt().requiresAck(), s_ack, s_idle) + } + } + is(s_ack) { // Wait for transaction to complete + io.inner.finish.ready := Bool(true) + when(io.inner.finish.valid) { state := s_idle } + } + } +} diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 4380e255..75eb897d 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -2,7 +2,6 @@ package uncore import Chisel._ -import scala.reflect.ClassTag case object CacheName extends Field[String] case object NSets extends Field[Int] @@ -11,6 +10,7 @@ case object BlockOffBits extends Field[Int] case object RowBits extends Field[Int] case object Replacer extends Field[() => ReplacementPolicy] case object AmoAluOperandBits extends Field[Int] +case object L2DirectoryRepresentation extends Field[DirectoryRepresentation] abstract trait CacheParameters extends UsesParameters { val nSets = params(NSets) @@ -140,7 +140,6 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { val write = Decoupled(new MetaWriteReq(rstVal.clone)).flip val resp = Vec.fill(nWays){rstVal.clone.asOutput} } - val metabits = rstVal.getWidth val rst_cnt = Reg(init=UInt(0, log2Up(nSets+1))) val rst = rst_cnt < UInt(nSets) val waddr = Mux(rst, rst_cnt, io.write.bits.idx) @@ -148,17 +147,14 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { val wmask = Mux(rst, SInt(-1), io.write.bits.way_en).toUInt when (rst) { rst_cnt := rst_cnt+UInt(1) } + val metabits = rstVal.getWidth val tag_arr = Mem(UInt(width = metabits*nWays), nSets, seqRead = true) when (rst || io.write.valid) { tag_arr.write(waddr, Fill(nWays, wdata), FillInterleaved(metabits, wmask)) } val tags = tag_arr(RegEnable(io.read.bits.idx, io.read.valid)) - for (w <- 0 until nWays) { - val m = tags(metabits*(w+1)-1, metabits*w) - io.resp(w) := rstVal.clone.fromBits(m) - } - + io.resp := io.resp.fromBits(tags) io.read.ready := !rst && !io.write.valid // so really this could be a 6T RAM io.write.ready := !rst } @@ -166,35 +162,17 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgentParameters { val idxMSB = idxBits-1 val idxLSB = 0 - val refillCyclesPerBeat = tlDataBits/rowBits - val refillCycles = refillCyclesPerBeat*tlDataBeats + val blockAddrBits = params(TLBlockAddrBits) + val refillCyclesPerBeat = outerDataBits/rowBits + val refillCycles = refillCyclesPerBeat*outerDataBeats require(refillCyclesPerBeat == 1) val amoAluOperandBits = params(AmoAluOperandBits) - require(amoAluOperandBits <= tlDataBits) + require(amoAluOperandBits <= innerDataBits) + require(rowBits == innerDataBits) // TODO: relax this by improving s_data_* states } -abstract class L2HellaCacheBundle extends TLBundle with L2HellaCacheParameters - -abstract class L2HellaCacheModule extends TLModule with L2HellaCacheParameters { - def connectDataBeatCounter[S <: HasTileLinkData](inc: Bool, data: S) = { - val (cnt, cnt_done) = - Counter(inc && data.hasMultibeatData(), tlDataBeats) - val done = (inc && !data.hasMultibeatData()) || cnt_done - (cnt, done) - } - def connectOutgoingDataBeatCounter[T <: HasTileLinkData : ClassTag](in: DecoupledIO[LogicalNetworkIO[T]]) = { - connectDataBeatCounter(in.fire(), in.bits.payload) - } - def connectIncomingDataBeatCounter[T <: HasTileLinkData](in: DecoupledIO[LogicalNetworkIO[T]]) = { - connectDataBeatCounter(in.fire(), in.bits.payload)._2 - } - def connectOutgoingDataBeatCounter[T <: HasTileLinkData](in: DecoupledIO[T]) = { - connectDataBeatCounter(in.fire(), in.bits) - } - def connectIncomingDataBeatCounter[T <: HasTileLinkData](in: ValidIO[T]) = { - connectDataBeatCounter(in.valid, in.bits)._2 - } -} +abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters +abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters trait HasL2Id extends Bundle with CoherenceAgentParameters { val id = UInt(width = log2Up(nTransactors + 1)) @@ -206,22 +184,29 @@ trait HasL2InternalRequestState extends L2HellaCacheBundle { val way_en = Bits(width = nWays) } -trait HasL2Data extends HasTileLinkData { +trait HasL2BeatAddr extends L2HellaCacheBundle { + val addr_beat = UInt(width = log2Up(refillCycles)) +} + +trait HasL2Data extends L2HellaCacheBundle + with HasL2BeatAddr { + val data = UInt(width = rowBits) def hasData(dummy: Int = 0) = Bool(true) - def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) + def hasMultibeatData(dummy: Int = 0) = Bool(refillCycles > 1) +} + +class L2Metadata extends Metadata with L2HellaCacheParameters { + val coh = new HierarchicalMetadata } object L2Metadata { - def apply(tag: Bits, coh: ManagerMetadata) = { + def apply(tag: Bits, coh: HierarchicalMetadata) = { val meta = new L2Metadata meta.tag := tag meta.coh := coh meta } } -class L2Metadata extends Metadata with L2HellaCacheParameters { - val coh = new ManagerMetadata -} class L2MetaReadReq extends MetaReadReq with HasL2Id { val tag = Bits(width = tagBits) @@ -250,7 +235,8 @@ class L2MetaRWIO extends L2HellaCacheBundle with HasL2MetaReadIO with HasL2MetaW class L2MetadataArray extends L2HellaCacheModule { val io = new L2MetaRWIO().flip - val meta = Module(new MetadataArray(() => L2Metadata(UInt(0), co.managerMetadataOnFlush))) + def onReset = L2Metadata(UInt(0), HierarchicalMetadata.onReset) + val meta = Module(new MetadataArray(onReset _)) meta.io.read <> io.read meta.io.write <> io.write @@ -259,7 +245,7 @@ class L2MetadataArray extends L2HellaCacheModule { def wayMap[T <: Data](f: Int => T) = Vec((0 until nWays).map(f)) val s1_clk_en = Reg(next = io.read.fire()) val s1_tag_eq_way = wayMap((w: Int) => meta.io.resp(w).tag === s1_tag) - val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && co.isValid(meta.io.resp(w).coh)).toBits + val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && meta.io.resp(w).coh.outer.isValid()).toBits val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) val s2_tag_match = s2_tag_match_way.orR val s2_hit_coh = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).coh, s1_clk_en))) @@ -281,7 +267,7 @@ class L2MetadataArray extends L2HellaCacheModule { } class L2DataReadReq extends L2HellaCacheBundle - with HasTileLinkBeatId + with HasL2BeatAddr with HasL2Id { val addr_idx = UInt(width = idxBits) val way_en = Bits(width = nWays) @@ -289,7 +275,7 @@ class L2DataReadReq extends L2HellaCacheBundle class L2DataWriteReq extends L2DataReadReq with HasL2Data { - val wmask = Bits(width = tlWriteMaskBits) + val wmask = Bits(width = rowBits/8) } class L2DataResp extends L2HellaCacheBundle with HasL2Id with HasL2Data @@ -305,7 +291,7 @@ trait HasL2DataWriteIO extends L2HellaCacheBundle { class L2DataRWIO extends L2HellaCacheBundle with HasL2DataReadIO with HasL2DataWriteIO -class L2DataArray extends L2HellaCacheModule { +class L2DataArray(delay: Int) extends L2HellaCacheModule { val io = new L2DataRWIO().flip val wmask = FillInterleaved(8, io.write.bits.wmask) @@ -320,23 +306,22 @@ class L2DataArray extends L2HellaCacheModule { reg_raddr := raddr } - io.resp.valid := ShiftRegister(io.read.fire(), 1) - io.resp.bits.id := ShiftRegister(io.read.bits.id, 1) - io.resp.bits.addr_beat := ShiftRegister(io.read.bits.addr_beat, 1) - io.resp.bits.data := array(reg_raddr) + io.resp.valid := ShiftRegister(io.read.fire(), delay+1) + io.resp.bits.id := ShiftRegister(io.read.bits.id, delay+1) + io.resp.bits.addr_beat := ShiftRegister(io.read.bits.addr_beat, delay+1) + io.resp.bits.data := ShiftRegister(array(reg_raddr), delay) io.read.ready := !io.write.valid io.write.ready := Bool(true) } -class L2HellaCacheBank(bankId: Int, innerId: String, outerId: String) extends - CoherenceAgent(innerId, outerId) with L2HellaCacheParameters { - +class L2HellaCacheBank(bankId: Int) extends HierarchicalCoherenceAgent + with L2HellaCacheParameters { require(isPow2(nSets)) require(isPow2(nWays)) - val tshrfile = Module(new TSHRFile(bankId, innerId, outerId)) + val tshrfile = Module(new TSHRFile(bankId)) val meta = Module(new L2MetadataArray) - val data = Module(new L2DataArray) + val data = Module(new L2DataArray(1)) tshrfile.io.inner <> io.inner tshrfile.io.meta <> meta.io @@ -345,45 +330,28 @@ class L2HellaCacheBank(bankId: Int, innerId: String, outerId: String) extends io.incoherent <> tshrfile.io.incoherent } +class TSHRFileIO extends HierarchicalTLIO { + val meta = new L2MetaRWIO + val data = new L2DataRWIO +} -class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCacheModule { - val io = new Bundle { - val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip - val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) - val incoherent = Vec.fill(nClients){Bool()}.asInput - val meta = new L2MetaRWIO - val data = new L2DataRWIO - } - - // Wiring helper funcs - def doOutputArbitration[T <: Data]( - out: DecoupledIO[T], - ins: Seq[DecoupledIO[T]], - count: Int = 1, - lock: T => Bool = (a: T) => Bool(true)) { - val arb = Module(new LockingRRArbiter(out.bits.clone, ins.size, count, lock)) - out <> arb.io.out - arb.io.in zip ins map { case (a, in) => a <> in } - } - - def doInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { - outs.map(_.bits := in.bits) - outs.zipWithIndex.map { case (o, i) => o.valid := in.valid && (UInt(i) === in.bits.id) } - } +class TSHRFile(bankId: Int) extends L2HellaCacheModule + with HasCoherenceAgentWiringHelpers { + val io = new TSHRFileIO // Create TSHRs for outstanding transactions val trackerList = (0 until nReleaseTransactors).map { id => - Module(new L2VoluntaryReleaseTracker(id, bankId, innerId, outerId)) + Module(new L2VoluntaryReleaseTracker(id, bankId)) } ++ (nReleaseTransactors until nTransactors).map { id => - Module(new L2AcquireTracker(id, bankId, innerId, outerId)) + Module(new L2AcquireTracker(id, bankId)) } - val wb = Module(new L2WritebackUnit(nTransactors, bankId, innerId, outerId)) + val wb = Module(new L2WritebackUnit(nTransactors, bankId)) doOutputArbitration(wb.io.wb.req, trackerList.map(_.io.wb.req)) doInputRouting(wb.io.wb.resp, trackerList.map(_.io.wb.resp)) // Propagate incoherence flags - (trackerList.map(_.io.tile_incoherent) :+ wb.io.tile_incoherent).map( _ := io.incoherent.toBits) + (trackerList.map(_.io.incoherent) :+ wb.io.incoherent).map( _ := io.incoherent.toBits) // Handle acquire transaction initiation val acquire = io.inner.acquire @@ -414,29 +382,14 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac } release.ready := Vec(releaseList.map(_.ready)).read(release_idx) - // Wire finished transaction acks - val finish = io.inner.finish - val finish_idx = finish.bits.payload.manager_xact_id - trackerList.zipWithIndex.map { case (t, i) => - t.io.inner.finish.valid := finish.valid && finish_idx === UInt(i) - } - trackerList.map(_.io.inner.finish.bits := finish.bits) - finish.ready := Vec(trackerList.map(_.io.inner.finish.ready)).read(finish_idx) - - // Wire probe requests to clients + // Wire probe requests and grant reply to clients, finish acks from clients doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe) :+ wb.io.inner.probe) - - // Wire grant reply to initiating client - doOutputArbitration( - io.inner.grant, - trackerList.map(_.io.inner.grant), - tlDataBeats, - (m: LogicalNetworkIO[Grant]) => m.payload.hasMultibeatData()) + doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) + doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish)) // Create an arbiter for the one memory port val outerList = trackerList.map(_.io.outer) :+ wb.io.outer - val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(outerList.size), - {case TLId => outerId}) + val outer_arb = Module(new TileLinkIOArbiterThatPassesId(outerList.size))(outerTLParams) outerList zip outer_arb.io.in map { case(out, arb) => out <> arb } io.outer <> outer_arb.io.out @@ -449,10 +402,495 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac doInputRouting(io.data.resp, trackerList.map(_.io.data.resp) :+ wb.io.data.resp) } + +class L2XactTrackerIO extends HierarchicalXactTrackerIO { + val data = new L2DataRWIO + val meta = new L2MetaRWIO + val wb = new L2WritebackIO +} + +abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { + def connectDataBeatCounter[S <: L2HellaCacheBundle](inc: Bool, data: S, beat: UInt, full_block: Bool) = { + if(data.refillCycles > 1) { + val (multi_cnt, multi_done) = Counter(inc, data.refillCycles) + (Mux(!full_block, beat, multi_cnt), Mux(!full_block, inc, multi_done)) + } else { (UInt(0), inc) } + } + def connectInternalDataBeatCounter[T <: HasL2BeatAddr]( + in: DecoupledIO[T], + beat: UInt = UInt(0), + full_block: Bool = Bool(true)) = { + connectDataBeatCounter(in.fire(), in.bits, beat, full_block) + } + def connectInternalDataBeatCounter[T <: HasL2Data]( + in: ValidIO[T], + full_block: Bool = Bool(true)) = { + connectDataBeatCounter(in.valid, in.bits, UInt(0), full_block)._2 + } +} + +class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTracker { + val io = new L2XactTrackerIO + + val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_inner_grant :: s_inner_finish :: Nil = Enum(UInt(), 7) + val state = Reg(init=s_idle) + + val xact_src = Reg(io.inner.release.bits.header.src.clone) + val xact = Reg(Bundle(new Release, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val xact_tag_match = Reg{ Bool() } + val xact_meta = Reg{ new L2Metadata } + val xact_way_en = Reg{ Bits(width = nWays) } + val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) } + val coh = xact_meta.coh + + val collect_irel_data = Reg(init=Bool(false)) + val irel_data_valid = Reg(init=Bits(0, width = innerDataBeats)) + val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val (write_data_cnt, write_data_done) = connectInternalDataBeatCounter(io.data.write) + + io.has_acquire_conflict := Bool(false) + io.has_acquire_match := Bool(false) + io.has_release_match := io.irel().isVoluntary() + + io.outer.acquire.valid := Bool(false) + io.outer.probe.ready := Bool(false) + io.outer.release.valid := Bool(false) + io.outer.grant.ready := Bool(false) + io.outer.finish.valid := Bool(false) + io.inner.acquire.ready := Bool(false) + io.inner.probe.valid := Bool(false) + io.inner.release.ready := Bool(false) + io.inner.grant.valid := Bool(false) + io.inner.finish.ready := Bool(false) + + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := xact_src + io.inner.grant.bits.payload := coh.inner.makeGrant(xact, UInt(trackerId)) + + io.data.read.valid := Bool(false) + io.data.write.valid := Bool(false) + io.data.write.bits.id := UInt(trackerId) + io.data.write.bits.way_en := xact_way_en + io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) + io.data.write.bits.addr_beat := write_data_cnt + io.data.write.bits.wmask := SInt(-1) + io.data.write.bits.data := data_buffer(write_data_cnt) + io.meta.read.valid := Bool(false) + io.meta.read.bits.id := UInt(trackerId) + io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) + io.meta.read.bits.tag := xact.addr_block >> UInt(idxBits) + io.meta.write.valid := Bool(false) + io.meta.write.bits.id := UInt(trackerId) + io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) + io.meta.write.bits.way_en := xact_way_en + io.meta.write.bits.data.tag := xact.addr_block >> UInt(idxBits) + io.meta.write.bits.data.coh.inner := xact_meta.coh.inner.onRelease(xact, xact_src) + io.meta.write.bits.data.coh.outer := xact_meta.coh.outer.onHit(M_XWR) // WB is a write + io.wb.req.valid := Bool(false) + + when(collect_irel_data) { + io.inner.release.ready := Bool(true) + when(io.inner.release.valid) { + data_buffer(io.irel().addr_beat) := io.irel().data + irel_data_valid(io.irel().addr_beat) := Bool(true) + } + when(irel_data_done) { collect_irel_data := Bool(false) } + } + + switch (state) { + is(s_idle) { + io.inner.release.ready := Bool(true) + when( io.inner.release.valid ) { + xact_src := io.inner.release.bits.header.src + xact := io.irel() + data_buffer(io.irel().addr_beat) := io.irel().data + collect_irel_data := io.irel().hasMultibeatData() + irel_data_valid := io.irel().hasData() << io.irel().addr_beat + state := s_meta_read + } + } + is(s_meta_read) { + io.meta.read.valid := Bool(true) + when(io.meta.read.ready) { state := s_meta_resp } + } + is(s_meta_resp) { + when(io.meta.resp.valid) { + xact_tag_match := io.meta.resp.bits.tag_match + xact_meta := io.meta.resp.bits.meta + xact_way_en := io.meta.resp.bits.way_en + state := Mux(io.meta.resp.bits.tag_match, + Mux(xact.hasData(), s_data_write, s_meta_write), + Mux(xact.requiresAck(), s_inner_grant, s_idle)) + } + } + is(s_data_write) { + io.data.write.valid := !collect_irel_data || irel_data_valid(write_data_cnt) + when(write_data_done) { state := s_meta_write } + } + is(s_meta_write) { + io.meta.write.valid := Bool(true) + when(io.meta.write.ready) { + state := Mux(xact.requiresAck(), s_inner_grant, s_idle) // Need a Grant.voluntaryAck? + } + } + is(s_inner_grant) { + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { + state := Mux(io.ignt().requiresAck(), s_inner_finish, s_idle) + } + } + is(s_inner_finish) { + io.inner.finish.ready := Bool(true) + when(io.inner.finish.valid) { state := s_idle } + } + } +} + +class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { + val io = new L2XactTrackerIO + + val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_probe :: s_outer_acquire :: s_outer_grant :: s_outer_finish :: s_data_read :: s_data_resp :: s_data_write :: s_inner_grant :: s_meta_write :: s_inner_finish :: Nil = Enum(UInt(), 15) + val state = Reg(init=s_idle) + + val xact_src = Reg(io.inner.acquire.bits.header.src.clone) + val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val data_buffer = Vec.fill(innerDataBeats+1) { // Extra entry holds AMO result + Reg(io.iacq().data.clone) + } + val xact_tag_match = Reg{ Bool() } + val xact_meta = Reg{ new L2Metadata } + val xact_way_en = Reg{ Bits(width = nWays) } + val pending_coh = Reg{ xact_meta.coh.clone } + val pending_finish = Reg{ io.outer.finish.bits.clone } + + val is_hit = xact_tag_match && xact_meta.coh.outer.isHit(xact.op_code()) + val do_allocate = xact.allocate() + val needs_writeback = !xact_tag_match && do_allocate && + xact_meta.coh.outer.requiresVoluntaryWriteback() + val needs_probes = xact_meta.coh.inner.requiresProbes(xact) + + val pending_coh_on_hit = HierarchicalMetadata( + io.meta.resp.bits.meta.coh.inner, + io.meta.resp.bits.meta.coh.outer.onHit(xact.op_code())) + val pending_coh_on_irel = HierarchicalMetadata( + pending_coh.inner.onRelease( + incoming = io.irel(), + src = io.inner.release.bits.header.src), + pending_coh.outer.onHit(M_XWR)) // WB is a write + val pending_coh_on_ognt = HierarchicalMetadata( + ManagerMetadata.onReset, + pending_coh.outer.onGrant(io.ognt(), xact.op_code())) + val pending_coh_on_ignt = HierarchicalMetadata( + pending_coh.inner.onGrant( + outgoing = io.ignt(), + dst = io.inner.grant.bits.header.dst), + pending_coh.outer) + + val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) + val pending_probes = Reg(init = Bits(0, width = nClients)) + val curr_p_id = PriorityEncoder(pending_probes) + val full_sharers = io.meta.resp.bits.meta.coh.inner.full() + val mask_self = Mux(xact.requiresSelfProbe(), + full_sharers | (UInt(1) << xact_src), + full_sharers & ~UInt(UInt(1) << xact_src, width = nClients)) + val mask_incoherent = mask_self & ~io.incoherent.toBits + + val collect_iacq_data = Reg(init=Bool(false)) + val iacq_data_valid = Reg(init=Bits(0, width = innerDataBeats)) + val irel_had_data = Reg(init = Bool(false)) + val ognt_had_data = Reg(init = Bool(false)) + val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) + val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) + val (ignt_data_cnt, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, xact.addr_beat) + val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire, xact.addr_beat) + val (read_data_cnt, read_data_done) = connectInternalDataBeatCounter(io.data.read, xact.addr_beat, !xact.isSubBlockType()) + val (write_data_cnt, write_data_done) = connectInternalDataBeatCounter(io.data.write, xact.addr_beat, !xact.isSubBlockType() || ognt_had_data || irel_had_data) + val resp_data_done = connectInternalDataBeatCounter(io.data.resp, !xact.isSubBlockType()) + + val amoalu = Module(new AMOALU) + amoalu.io.addr := xact.addr() + amoalu.io.cmd := xact.op_code() + amoalu.io.typ := xact.op_size() + amoalu.io.lhs := io.data.resp.bits.data //default + amoalu.io.rhs := data_buffer(0) // default + + // TODO: figure out how to merge the following three versions of this func + def mergeDataInternal[T <: HasL2Data](buffer: Vec[UInt], incoming: T) { + val old_data = incoming.data + val new_data = buffer(incoming.addr_beat) + val amoOpSz = UInt(amoAluOperandBits) + val offset = xact.addr_byte()(innerByteAddrBits-1, log2Up(amoAluOperandBits/8)) + amoalu.io.lhs := old_data >> offset*amoOpSz + amoalu.io.rhs := new_data >> offset*amoOpSz + val wmask = + Mux(xact.is(Acquire.putAtomicType), + FillInterleaved(amoAluOperandBits, UIntToOH(offset)), + Mux(xact.is(Acquire.putBlockType) || xact.is(Acquire.putType), + FillInterleaved(8, xact.write_mask()), + UInt(0, width = innerDataBits))) + buffer(incoming.addr_beat) := ~wmask & old_data | wmask & + Mux(xact.is(Acquire.putAtomicType), amoalu.io.out << offset*amoOpSz, new_data) + when(xact.is(Acquire.putAtomicType)) { buffer(innerDataBeats) := old_data } // For AMO result + } + def mergeDataInner[T <: HasTileLinkData](buffer: Vec[UInt], incoming: T) = mergeDataOuter(buffer, incoming) + def mergeDataOuter[T <: HasTileLinkData](buffer: Vec[UInt], incoming: T) { + val old_data = incoming.data + val new_data = buffer(incoming.addr_beat) + val amoOpSz = UInt(amoAluOperandBits) + val offset = xact.addr_byte()(innerByteAddrBits-1, log2Up(amoAluOperandBits/8)) + amoalu.io.lhs := old_data >> offset*amoOpSz + amoalu.io.rhs := new_data >> offset*amoOpSz + val wmask = + Mux(xact.is(Acquire.putAtomicType), + FillInterleaved(amoAluOperandBits, UIntToOH(offset)), + Mux(xact.is(Acquire.putBlockType) || xact.is(Acquire.putType), + FillInterleaved(8, xact.write_mask()), + UInt(0, width = innerDataBits))) + buffer(incoming.addr_beat) := ~wmask & old_data | wmask & + Mux(xact.is(Acquire.putAtomicType), amoalu.io.out << offset*amoOpSz, new_data) + when(xact.is(Acquire.putAtomicType)) { buffer(innerDataBeats) := old_data } // For AMO result + } + + //TODO: Allow hit under miss for stores + val in_same_set = xact.addr_block(idxMSB,idxLSB) === + io.iacq().addr_block(idxMSB,idxLSB) + io.has_acquire_conflict := (xact.conflicts(io.iacq()) || in_same_set) && + (state != s_idle) && + !collect_iacq_data + io.has_acquire_match := xact.conflicts(io.iacq()) && + collect_iacq_data + io.has_release_match := !io.irel().isVoluntary() && + (xact.addr_block === io.irel().addr_block) && + (state === s_probe) + + // If we're allocating in this cache, we can use the current metadata + // to make an appropriate custom Acquire, otherwise we copy over the + // built-in Acquire from the inner TL to the outer TL + io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.payload := Mux(do_allocate, + xact_meta.coh.outer.makeAcquire( + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block, + op_code = xact.op_code()), + Bundle(Acquire(xact))(outerTLParams)) + io.outer.acquire.bits.header.src := UInt(bankId) + io.outer.probe.ready := Bool(false) + io.outer.release.valid := Bool(false) + io.outer.grant.ready := Bool(false) + io.outer.finish.valid := Bool(false) + io.outer.finish.bits := pending_finish + val pending_finish_on_ognt = io.ognt().makeFinish() + + io.inner.probe.valid := Bool(false) + io.inner.probe.bits.header.src := UInt(bankId) + io.inner.probe.bits.header.dst := curr_p_id + io.inner.probe.bits.payload := pending_coh.inner.makeProbe(xact) + + io.inner.grant.valid := Bool(false) + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := xact_src + io.inner.grant.bits.payload := pending_coh.inner.makeGrant( + acq = xact, + manager_xact_id = UInt(trackerId), + addr_beat = ignt_data_cnt, + data = Mux(xact.is(Acquire.putAtomicType), + data_buffer(innerDataBeats), + data_buffer(ignt_data_cnt))) + + io.inner.acquire.ready := Bool(false) + io.inner.release.ready := Bool(false) + io.inner.finish.ready := Bool(false) + + io.data.read.valid := Bool(false) + io.data.read.bits.id := UInt(trackerId) + io.data.read.bits.way_en := xact_way_en + io.data.read.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) + io.data.read.bits.addr_beat := read_data_cnt + io.data.write.valid := Bool(false) + io.data.write.bits.id := UInt(trackerId) + io.data.write.bits.way_en := xact_way_en + io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) + io.data.write.bits.addr_beat := write_data_cnt + io.data.write.bits.wmask := SInt(-1) + io.data.write.bits.data := data_buffer(write_data_cnt) + io.meta.read.valid := Bool(false) + io.meta.read.bits.id := UInt(trackerId) + io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) + io.meta.read.bits.tag := xact.addr_block >> UInt(idxBits) + io.meta.write.valid := Bool(false) + io.meta.write.bits.id := UInt(trackerId) + io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) + io.meta.write.bits.way_en := xact_way_en + io.meta.write.bits.data.tag := xact.addr_block >> UInt(idxBits) + io.meta.write.bits.data.coh := pending_coh + + io.wb.req.valid := Bool(false) + io.wb.req.bits.addr_block := Cat(xact_meta.tag, xact.addr_block(idxMSB,idxLSB)) + io.wb.req.bits.coh := xact_meta.coh + io.wb.req.bits.way_en := xact_way_en + io.wb.req.bits.id := UInt(trackerId) + + when(collect_iacq_data) { + io.inner.acquire.ready := Bool(true) + when(io.inner.acquire.valid) { + data_buffer(io.iacq().addr_beat) := io.iacq().data + iacq_data_valid(io.iacq().addr_beat) := Bool(true) + } + when(iacq_data_done) { collect_iacq_data := Bool(false) } + } + + switch (state) { + is(s_idle) { + io.inner.acquire.ready := Bool(true) + when(io.inner.acquire.valid) { + xact_src := io.inner.acquire.bits.header.src + xact := io.iacq() + data_buffer(io.iacq().addr_beat) := io.iacq().data + collect_iacq_data := io.iacq().hasMultibeatData() + iacq_data_valid := io.iacq().hasData() << io.iacq().addr_beat + irel_had_data := Bool(false) + ognt_had_data := Bool(false) + state := s_meta_read + } + } + is(s_meta_read) { + io.meta.read.valid := Bool(true) + when(io.meta.read.ready) { state := s_meta_resp } + } + is(s_meta_resp) { + when(io.meta.resp.valid) { + xact_tag_match := io.meta.resp.bits.tag_match + xact_meta := io.meta.resp.bits.meta + xact_way_en := io.meta.resp.bits.way_en + pending_coh := io.meta.resp.bits.meta.coh + val _coh = io.meta.resp.bits.meta.coh + val _tag_match = io.meta.resp.bits.tag_match + val _is_hit = _tag_match && _coh.outer.isHit(xact.op_code()) + val _needs_writeback = !_tag_match && do_allocate && + _coh.outer.requiresVoluntaryWriteback() + val _needs_probes = _tag_match && _coh.inner.requiresProbes(xact) + when(_is_hit) { pending_coh := pending_coh_on_hit } + when(_needs_probes) { + pending_probes := mask_incoherent(nCoherentClients-1,0) + release_count := PopCount(mask_incoherent(nCoherentClients-1,0)) + } + state := Mux(_tag_match, + Mux(_needs_probes, s_probe, Mux(_is_hit, s_data_read, s_outer_acquire)), // Probe, hit or upgrade + Mux(_needs_writeback, s_wb_req, s_outer_acquire)) // Evict ifneedbe + } + } + is(s_wb_req) { + io.wb.req.valid := Bool(true) + when(io.wb.req.ready) { state := s_wb_resp } + } + is(s_wb_resp) { + when(io.wb.resp.valid) { state := s_outer_acquire } + } + is(s_probe) { + // Send probes + io.inner.probe.valid := pending_probes != UInt(0) + when(io.inner.probe.ready) { + pending_probes := pending_probes & ~UIntToOH(curr_p_id) + } + // Handle releases, which may have data being written back + io.inner.release.ready := Bool(true) + when(io.inner.release.valid) { + pending_coh := pending_coh_on_irel + // Handle released dirty data + //TODO: make sure cacq data is actually present before accpeting + // release data to merge! + when(io.irel().hasData()) { + irel_had_data := Bool(true) + mergeDataInner(data_buffer, io.irel()) + } + // We don't decrement release_count until we've received all the data beats. + when(!io.irel().hasMultibeatData() || irel_data_done) { + release_count := release_count - UInt(1) + } + } + when(release_count === UInt(0)) { + state := Mux(is_hit, Mux(irel_had_data, s_data_write, s_data_read), s_outer_acquire) + } + } + is(s_outer_acquire) { + io.outer.acquire.valid := !iacq_data_done // collect all data before refilling + when(oacq_data_done) { + state := s_outer_grant + } + } + is(s_outer_grant) { + io.outer.grant.ready := Bool(true) + when(io.outer.grant.valid) { + when(io.ognt().hasData()) { + mergeDataOuter(data_buffer, io.ognt()) + ognt_had_data := Bool(true) + } + when(ognt_data_done) { + pending_coh := pending_coh_on_ognt + when(io.ognt().requiresAck()) { + pending_finish.payload := pending_finish_on_ognt + pending_finish.header.dst := io.outer.grant.bits.header.src + state := s_outer_finish + }.otherwise { + state := Mux(!do_allocate, s_inner_grant, + Mux(io.ognt().hasData(), s_data_write, s_data_read)) + } + } + } + } + is(s_outer_finish) { + io.outer.finish.valid := Bool(true) + when(io.outer.finish.ready) { + state := Mux(!do_allocate, s_inner_grant, + Mux(ognt_had_data, s_data_write, s_data_read)) + } + } + is(s_data_read) { + io.data.read.valid := !collect_iacq_data || iacq_data_valid(read_data_cnt) + when(io.data.resp.valid) { + mergeDataInternal(data_buffer, io.data.resp.bits) + } + when(read_data_done) { state := s_data_resp } + } + is(s_data_resp) { + when(io.data.resp.valid) { + mergeDataInternal(data_buffer, io.data.resp.bits) + } + when(resp_data_done) { + state := Mux(xact.hasData(), s_data_write, s_inner_grant) + } + } + is(s_data_write) { + io.data.write.valid := Bool(true) + when(write_data_done) { state := s_inner_grant } + } + is(s_inner_grant) { + io.inner.grant.valid := Bool(true) + when(ignt_data_done) { + val meta = pending_coh_on_ignt != xact_meta.coh + when(meta) { pending_coh := pending_coh_on_ignt } + state := Mux(meta, s_meta_write, + Mux(io.ignt().requiresAck(), s_inner_finish, s_idle)) + } + } + is(s_meta_write) { + io.meta.write.valid := Bool(true) + when(io.meta.write.ready) { + state := Mux(io.ignt().requiresAck(), s_inner_finish, s_idle) + } + } + is(s_inner_finish) { + io.inner.finish.ready := Bool(true) + when(io.inner.finish.valid) { state := s_idle } + } + } +} + class L2WritebackReq extends L2HellaCacheBundle with HasL2Id { - val addr_block = UInt(width = tlBlockAddrBits) - val coh = new ManagerMetadata + val addr_block = UInt(width = blockAddrBits) // TODO: assumes same block size + val coh = new HierarchicalMetadata val way_en = Bits(width = nWays) } @@ -463,58 +901,63 @@ class L2WritebackIO extends L2HellaCacheBundle { val resp = Valid(new L2WritebackResp).flip } -class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2HellaCacheModule { - val io = new Bundle { - val wb = new L2WritebackIO().flip - val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip - val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) - val tile_incoherent = Bits(INPUT, nClients) - val has_release_match = Bool(OUTPUT) - val data = new L2DataRWIO - } - val cacq = io.inner.acquire.bits - val crel = io.inner.release.bits - val cgnt = io.inner.grant.bits - val mgnt = io.outer.grant.bits +class L2WritebackUnitIO extends HierarchicalXactTrackerIO { + val wb = new L2WritebackIO().flip + val data = new L2DataRWIO +} - val s_idle :: s_probe :: s_data_read :: s_data_resp :: s_outer_write :: Nil = Enum(UInt(), 5) +class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { + val io = new L2WritebackUnitIO + + val s_idle :: s_probe :: s_data_read :: s_data_resp :: s_outer_release :: s_outer_grant :: s_outer_finish :: s_wb_resp :: Nil = Enum(UInt(), 8) val state = Reg(init=s_idle) - val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone) - val xact_coh = Reg{ new ManagerMetadata } + val xact_addr_block = Reg(io.wb.req.bits.addr_block.clone) + val xact_coh = Reg{ new HierarchicalMetadata } val xact_way_en = Reg{ Bits(width = nWays) } - val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } + val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) } val xact_id = Reg{ UInt() } + val pending_finish = Reg{ io.outer.finish.bits.clone } - val crel_had_data = Reg(init = Bool(false)) + val irel_had_data = Reg(init = Bool(false)) val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) - val pending_probes = Reg(init = co.dir.flush) - val curr_p_id = co.dir.next(pending_probes) + val pending_probes = Reg(init = Bits(0, width = nClients)) + val curr_p_id = PriorityEncoder(pending_probes) - val crel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val (macq_data_cnt, macq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire) - val (read_data_cnt, read_data_done) = Counter(io.data.read.fire(), tlDataBeats) - val resp_data_done = connectIncomingDataBeatCounter(io.data.resp) + val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val (orel_data_cnt, orel_data_done) = connectOutgoingDataBeatCounter(io.outer.release) + val (read_data_cnt, read_data_done) = connectInternalDataBeatCounter(io.data.read) + val resp_data_done = connectInternalDataBeatCounter(io.data.resp) - io.has_release_match := !crel.payload.isVoluntary() && - crel.payload.conflicts(xact_addr_block) && + val pending_icoh_on_irel = xact_coh.inner.onRelease( + incoming = io.irel(), + src = io.inner.release.bits.header.src) + + io.has_acquire_conflict := Bool(false) + io.has_acquire_match := Bool(false) + io.has_release_match := !io.irel().isVoluntary() && + io.irel().conflicts(xact_addr_block) && (state === s_probe) - val next_coh_on_rel = co.managerMetadataOnRelease(crel.payload, xact_coh, crel.header.src) - io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits.payload := Bundle(UncachedWriteBlock( + io.outer.probe.ready := Bool(false) + io.outer.release.valid := Bool(false) // default + io.outer.release.bits.payload := xact_coh.outer.makeVoluntaryWriteback( client_xact_id = UInt(trackerId), addr_block = xact_addr_block, - addr_beat = macq_data_cnt, - data = xact_data(macq_data_cnt)), - { case TLId => outerId }) - io.outer.grant.ready := Bool(false) // Never gets mgnts + addr_beat = orel_data_cnt, + data = data_buffer(orel_data_cnt)) + io.outer.release.bits.header.src := UInt(bankId) + io.outer.grant.ready := Bool(false) // default + io.outer.finish.valid := Bool(false) // default + io.outer.finish.bits := pending_finish + val pending_finish_on_ognt = io.ognt().makeFinish() io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe.onVoluntaryWriteback(xact_coh, xact_addr_block) + io.inner.probe.bits.payload := + xact_coh.inner.makeProbeForVoluntaryWriteback(xact_addr_block) io.inner.grant.valid := Bool(false) io.inner.acquire.ready := Bool(false) @@ -540,528 +983,75 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str xact_coh := io.wb.req.bits.coh xact_way_en := io.wb.req.bits.way_en xact_id := io.wb.req.bits.id + irel_had_data := Bool(false) val coh = io.wb.req.bits.coh - val needs_probes = co.requiresProbesOnVoluntaryWriteback(coh) + val needs_probes = coh.inner.requiresProbesOnVoluntaryWriteback() when(needs_probes) { - val mask_incoherent = co.dir.full(coh.sharers) & ~io.tile_incoherent - pending_probes := mask_incoherent - release_count := co.dir.count(mask_incoherent) - crel_had_data := Bool(false) + val mask_incoherent = coh.inner.full() & ~io.incoherent.toBits + pending_probes := mask_incoherent(nCoherentClients-1,0) + release_count := PopCount(mask_incoherent(nCoherentClients-1,0)) } state := Mux(needs_probes, s_probe, s_data_read) } } is(s_probe) { // Send probes - io.inner.probe.valid := !co.dir.none(pending_probes) + io.inner.probe.valid := pending_probes != UInt(0) when(io.inner.probe.ready) { - pending_probes := co.dir.pop(pending_probes, curr_p_id) + pending_probes := pending_probes & ~UIntToOH(curr_p_id) } // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) when(io.inner.release.valid) { - xact_coh := next_coh_on_rel + xact_coh.inner := pending_icoh_on_irel // Handle released dirty data - when(crel.payload.hasData()) { - crel_had_data := Bool(true) - xact_data(crel.payload.addr_beat) := crel.payload.data + when(io.irel().hasData()) { + irel_had_data := Bool(true) + data_buffer(io.irel().addr_beat) := io.irel().data } // We don't decrement release_count until we've received all the data beats. - when(!crel.payload.hasData() || crel_data_done) { + when(!io.irel().hasData() || irel_data_done) { release_count := release_count - UInt(1) } } when(release_count === UInt(0)) { - state := Mux(crel_had_data, s_outer_write, s_data_read) + state := Mux(irel_had_data, s_outer_release, s_data_read) } } is(s_data_read) { io.data.read.valid := Bool(true) - when(io.data.resp.valid) { xact_data(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } + when(io.data.resp.valid) { data_buffer(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } when(read_data_done) { state := s_data_resp } } is(s_data_resp) { - when(io.data.resp.valid) { xact_data(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } - when(resp_data_done) { state := s_outer_write } + when(io.data.resp.valid) { data_buffer(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } + when(resp_data_done) { state := s_outer_release } } - is(s_outer_write) { - io.outer.acquire.valid := Bool(true) - when(macq_data_done) { - io.wb.resp.valid := Bool(true) - state := s_idle + is(s_outer_release) { + io.outer.release.valid := Bool(true) + when(orel_data_done) { + state := Mux(io.orel().requiresAck(), s_outer_grant, s_wb_resp) } } - } -} - -abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCacheModule { - val io = new Bundle { - val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip - val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) - val tile_incoherent = Bits(INPUT, nClients) - val has_acquire_conflict = Bool(OUTPUT) - val has_acquire_match = Bool(OUTPUT) - val has_release_match = Bool(OUTPUT) - val data = new L2DataRWIO - val meta = new L2MetaRWIO - val wb = new L2WritebackIO - } - - val cacq = io.inner.acquire.bits - val crel = io.inner.release.bits - val cgnt = io.inner.grant.bits - val cack = io.inner.finish.bits - val mgnt = io.outer.grant.bits -} - -class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_grant :: s_ack :: Nil = Enum(UInt(), 7) - val state = Reg(init=s_idle) - - val xact_src = Reg(io.inner.release.bits.header.src.clone) - val xact_r_type = Reg(io.inner.release.bits.payload.r_type) - val xact_addr_block = Reg(io.inner.release.bits.payload.addr_block.clone) - val xact_addr_beat = Reg(io.inner.release.bits.payload.addr_beat.clone) - val xact_client_xact_id = Reg(io.inner.release.bits.payload.client_xact_id.clone) - val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) } - val xact_tag_match = Reg{ Bool() } - val xact_meta = Reg{ new L2Metadata } - val xact_way_en = Reg{ Bits(width = nWays) } - val xact = Release( - voluntary = Bool(true), - r_type = xact_r_type, - client_xact_id = xact_client_xact_id, - addr_block = xact_addr_block) - - val collect_crel_data = Reg(init=Bool(false)) - val crel_data_valid = Reg(init=Bits(0, width = tlDataBeats)) - val crel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val (write_data_cnt, write_data_done) = connectOutgoingDataBeatCounter(io.data.write) - - io.has_acquire_conflict := Bool(false) - io.has_acquire_match := Bool(false) - io.has_release_match := crel.payload.isVoluntary() - - io.outer.grant.ready := Bool(false) - io.outer.acquire.valid := Bool(false) - io.inner.acquire.ready := Bool(false) - io.inner.probe.valid := Bool(false) - io.inner.release.ready := Bool(false) - io.inner.grant.valid := Bool(false) - io.inner.finish.ready := Bool(false) - - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := xact.makeGrant(UInt(trackerId), xact_meta.coh) - - io.data.read.valid := Bool(false) - io.data.write.valid := Bool(false) - io.data.write.bits.id := UInt(trackerId) - io.data.write.bits.way_en := xact_way_en - io.data.write.bits.addr_idx := xact_addr_block(idxMSB,idxLSB) - io.data.write.bits.addr_beat := write_data_cnt - io.data.write.bits.wmask := SInt(-1) - io.data.write.bits.data := xact_data(write_data_cnt) - io.meta.read.valid := Bool(false) - io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact_addr_block(idxMSB,idxLSB) - io.meta.read.bits.tag := xact_addr_block >> UInt(idxBits) - io.meta.write.valid := Bool(false) - io.meta.write.bits.id := UInt(trackerId) - io.meta.write.bits.idx := xact_addr_block(idxMSB,idxLSB) - io.meta.write.bits.way_en := xact_way_en - io.meta.write.bits.data.tag := xact_addr_block >> UInt(idxBits) - io.meta.write.bits.data.coh := co.managerMetadataOnRelease(xact, - xact_meta.coh, - xact_src) - io.wb.req.valid := Bool(false) - - when(collect_crel_data) { - io.inner.release.ready := Bool(true) - when(io.inner.release.valid) { - xact_data(crel.payload.addr_beat) := crel.payload.data - crel_data_valid(crel.payload.addr_beat) := Bool(true) - } - when(crel_data_done) { collect_crel_data := Bool(false) } - } - - switch (state) { - is(s_idle) { - io.inner.release.ready := Bool(true) - when( io.inner.release.valid ) { - xact_src := crel.header.src - xact_r_type := crel.payload.r_type - xact_addr_block := crel.payload.addr_block - xact_addr_beat := crel.payload.addr_beat - xact_client_xact_id := crel.payload.client_xact_id - xact_data(UInt(0)) := crel.payload.data - collect_crel_data := crel.payload.hasMultibeatData() - crel_data_valid := Bits(1) - state := s_meta_read + is(s_outer_grant) { + io.outer.grant.ready := Bool(true) + when(io.outer.grant.valid) { + when(io.ognt().requiresAck()) { + pending_finish.payload := pending_finish_on_ognt + pending_finish.header.dst := io.outer.grant.bits.header.src + state := s_outer_finish + }.otherwise { + state := s_wb_resp + } } } - is(s_meta_read) { - io.meta.read.valid := Bool(true) - when(io.meta.read.ready) { state := s_meta_resp } - } - is(s_meta_resp) { - when(io.meta.resp.valid) { - xact_tag_match := io.meta.resp.bits.tag_match - xact_meta := io.meta.resp.bits.meta - xact_way_en := io.meta.resp.bits.way_en - state := Mux(io.meta.resp.bits.tag_match, - Mux(xact.hasData(), s_data_write, s_meta_write), - Mux(xact.requiresAck(), s_grant, s_idle)) - } - } - is(s_data_write) { - io.data.write.valid := !collect_crel_data || crel_data_valid(write_data_cnt) - when(write_data_done) { state := s_meta_write } - } - is(s_meta_write) { - io.meta.write.valid := Bool(true) - when(io.meta.write.ready) { - state := Mux(xact.requiresAck(), s_grant, s_idle) // Need a Grant.voluntaryAck? - } - } - is(s_grant) { - io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { - state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) - } - } - is(s_ack) { - // TODO: This state is unnecessary if no client will ever issue the - // pending Acquire that caused this writeback until it receives the - // Grant.voluntaryAck for this writeback - io.inner.finish.ready := Bool(true) - when(io.inner.finish.valid) { state := s_idle } - } - } -} - - -class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_probe :: s_outer_read :: s_outer_resp :: s_data_read :: s_data_resp :: s_data_write :: s_meta_write :: s_grant :: s_ack :: Nil = Enum(UInt(), 14) - val state = Reg(init=s_idle) - - val xact_src = Reg(io.inner.acquire.bits.header.src.clone) - val xact_builtin_type = Reg(io.inner.acquire.bits.payload.builtin_type.clone) - val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone) - val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone) - val xact_addr_beat = Reg(io.inner.acquire.bits.payload.addr_beat.clone) - val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone) - val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone) - val xact_data = Vec.fill(tlDataBeats+1) { // Extra entry holds AMO result - Reg(io.inner.acquire.bits.payload.data.clone) - } - val xact_tag_match = Reg{ Bool() } - val xact_meta = Reg{ new L2Metadata } - val xact_way_en = Reg{ Bits(width = nWays) } - val xact = Acquire( - builtin_type = xact_builtin_type, - a_type = xact_a_type, - client_xact_id = xact_client_xact_id, - addr_block = xact_addr_block, - addr_beat = xact_addr_beat, - data = UInt(0), - subblock = xact_subblock) - - val collect_cacq_data = Reg(init=Bool(false)) - val cacq_data_valid = Reg(init=Bits(0, width = tlDataBeats)) - val crel_had_data = Reg(init = Bool(false)) - val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) - val pending_probes = Reg(init = UInt(0, width = nCoherentClients)) - val curr_p_id = co.dir.next(pending_probes) - val full_sharers = co.dir.full(io.meta.resp.bits.meta.coh.sharers) - val mask_self = Mux(xact.requiresSelfProbe(), - full_sharers | (UInt(1) << xact_src), - full_sharers & ~UInt(UInt(1) << xact_src, width = nClients)) - val mask_incoherent = mask_self & ~io.tile_incoherent - - val cacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) - val crel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val (macq_data_cnt, macq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire) - val mgnt_data_done = connectIncomingDataBeatCounter(io.outer.grant) - val cgnt_data_cnt = Reg(init = UInt(0, width = tlBeatAddrBits+1)) - val cgnt_data_max = Reg(init = UInt(0, width = tlBeatAddrBits+1)) - val read_data_cnt = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) - val read_data_max = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) - val write_data_cnt = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) - val write_data_max = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) - val resp_data_cnt = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) - val resp_data_max = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) - - val needs_writeback = !xact_tag_match && co.isValid(xact_meta.coh) // TODO: dirty bit - val is_hit = xact_tag_match && co.isHit(xact, xact_meta.coh) - val needs_probes = co.requiresProbes(xact, xact_meta.coh) - //val do_allocate = !xact_builtin_type || xact.allocate() - - val amoalu = Module(new AMOALU) - amoalu.io.addr := xact.addr() - amoalu.io.cmd := xact.op_code() - amoalu.io.typ := xact.op_size() - amoalu.io.lhs := io.data.resp.bits.data //default - amoalu.io.rhs := xact.data(0) // default - - def mergeData[T <: HasTileLinkData](buffer: Vec[UInt], incoming: T) { - val old_data = incoming.data - val new_data = buffer(incoming.addr_beat) - val amoOpSz = UInt(amoAluOperandBits) - val offset = xact.addr_byte()(tlByteAddrBits-1, log2Up(amoAluOperandBits/8)) - amoalu.io.lhs := old_data >> offset*amoOpSz - amoalu.io.rhs := new_data >> offset*amoOpSz - val wmask = - Mux(xact.is(Acquire.uncachedAtomic), - FillInterleaved(amoAluOperandBits, UIntToOH(offset)), - Mux(xact.is(Acquire.uncachedWriteBlock) || xact.is(Acquire.uncachedWrite), - FillInterleaved(8, xact.write_mask()), - UInt(0, width = tlDataBits))) - buffer(incoming.addr_beat) := ~wmask & old_data | wmask & - Mux(xact.is(Acquire.uncachedAtomic), amoalu.io.out << offset*amoOpSz, new_data) - when(xact.is(Acquire.uncachedAtomic)) { buffer(tlDataBeats) := old_data } // For AMO result - } - - //TODO: Allow hit under miss for stores - val in_same_set = xact.addr_block(idxMSB,idxLSB) === - cacq.payload.addr_block(idxMSB,idxLSB) - io.has_acquire_conflict := (xact.conflicts(cacq.payload) || in_same_set) && - (state != s_idle) && - !collect_cacq_data - io.has_acquire_match := xact.conflicts(cacq.payload) && - collect_cacq_data - io.has_release_match := !crel.payload.isVoluntary() && - (xact.addr_block === crel.payload.addr_block) && - (state === s_probe) - - val next_coh_on_rel = co.managerMetadataOnRelease( - incoming = crel.payload, - meta = xact_meta.coh, - src = crel.header.src) - val next_coh_on_gnt = co.managerMetadataOnGrant( - outgoing = cgnt.payload, - meta = xact_meta.coh, - dst = cgnt.header.dst) - - val outer_write = Bundle(UncachedWriteBlock( - client_xact_id = UInt(trackerId), - addr_block = xact_addr_block, - addr_beat = macq_data_cnt, - data = xact_data(macq_data_cnt)), - { case TLId => outerId }) - val outer_read = Bundle(UncachedReadBlock( - client_xact_id = UInt(trackerId), - addr_block = xact_addr_block), - { case TLId => outerId }) - - io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits.payload := outer_read //default - io.outer.grant.ready := Bool(true) //grant.data -> xact.data - - io.inner.probe.valid := Bool(false) - io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := xact.makeProbe(xact_meta.coh) - - io.inner.grant.valid := Bool(false) - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := xact.makeGrant( - manager_xact_id = UInt(trackerId), - meta = xact_meta.coh, - addr_beat = cgnt_data_cnt, - data = Mux(xact.is(Acquire.uncachedAtomic), - xact_data(tlDataBeats), - xact_data(cgnt_data_cnt))) - - io.inner.acquire.ready := Bool(false) - io.inner.release.ready := Bool(false) - io.inner.finish.ready := Bool(false) - - io.data.read.valid := Bool(false) - io.data.read.bits.id := UInt(trackerId) - io.data.read.bits.way_en := xact_way_en - io.data.read.bits.addr_idx := xact_addr_block(idxMSB,idxLSB) - io.data.read.bits.addr_beat := read_data_cnt - io.data.write.valid := Bool(false) - io.data.write.bits.id := UInt(trackerId) - io.data.write.bits.way_en := xact_way_en - io.data.write.bits.addr_idx := xact_addr_block(idxMSB,idxLSB) - io.data.write.bits.addr_beat := write_data_cnt - io.data.write.bits.wmask := SInt(-1) - io.data.write.bits.data := xact_data(write_data_cnt) - io.meta.read.valid := Bool(false) - io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact_addr_block(idxMSB,idxLSB) - io.meta.read.bits.tag := xact_addr_block >> UInt(idxBits) - io.meta.write.valid := Bool(false) - io.meta.write.bits.id := UInt(trackerId) - io.meta.write.bits.idx := xact_addr_block(idxMSB,idxLSB) - io.meta.write.bits.way_en := xact_way_en - io.meta.write.bits.data.tag := xact_addr_block >> UInt(idxBits) - io.meta.write.bits.data.coh := next_coh_on_gnt - - io.wb.req.valid := Bool(false) - io.wb.req.bits.addr_block := Cat(xact_meta.tag, xact_addr_block(idxMSB,idxLSB)) - io.wb.req.bits.coh := xact_meta.coh - io.wb.req.bits.way_en := xact_way_en - io.wb.req.bits.id := UInt(trackerId) - - when(collect_cacq_data) { - io.inner.acquire.ready := Bool(true) - when(io.inner.acquire.valid) { - xact_data(cacq.payload.addr_beat) := cacq.payload.data - cacq_data_valid(cacq.payload.addr_beat) := Bool(true) - } - when(cacq_data_done) { collect_cacq_data := Bool(false) } - } - - switch (state) { - is(s_idle) { - io.inner.acquire.ready := Bool(true) - when( io.inner.acquire.valid ) { - xact_builtin_type := cacq.payload.builtin_type - xact_a_type := cacq.payload.a_type - xact_addr_block := cacq.payload.addr_block - xact_addr_beat := cacq.payload.addr_beat - xact_client_xact_id := cacq.payload.client_xact_id - xact_data(UInt(0)) := cacq.payload.data - xact_subblock := cacq.payload.subblock - xact_src := cacq.header.src - collect_cacq_data := cacq.payload.hasMultibeatData() - state := s_meta_read - } - } - is(s_meta_read) { - io.meta.read.valid := Bool(true) - when(io.meta.read.ready) { state := s_meta_resp } - } - is(s_meta_resp) { - when(io.meta.resp.valid) { - xact_tag_match := io.meta.resp.bits.tag_match - xact_meta := io.meta.resp.bits.meta - xact_way_en := io.meta.resp.bits.way_en - val coh = io.meta.resp.bits.meta.coh - val _tag_match = io.meta.resp.bits.tag_match - val _needs_writeback = !_tag_match && co.isValid(coh) //TODO: dirty bit - val _needs_probes = _tag_match && co.requiresProbes(xact, coh) - val _is_hit = _tag_match && co.isHit(xact, coh) - val full_block = !xact.builtin_type || - xact.hasMultibeatData() || - cgnt.payload.hasMultibeatData() - read_data_cnt := Mux(full_block, UInt(0), xact_addr_beat) - read_data_max := Mux(full_block, UInt(refillCycles-1), xact_addr_beat) - write_data_cnt := Mux(full_block || !_is_hit, UInt(0), xact_addr_beat) - write_data_max := Mux(full_block || !_is_hit, UInt(refillCycles-1), xact_addr_beat) - resp_data_cnt := Mux(full_block, UInt(0), xact_addr_beat) - resp_data_max := Mux(full_block, UInt(refillCycles-1), xact_addr_beat) - cgnt_data_cnt := Mux(full_block, UInt(0), xact_addr_beat) - cgnt_data_max := Mux(full_block, UInt(tlDataBeats-1), xact_addr_beat) - when(_needs_probes) { - pending_probes := mask_incoherent(nCoherentClients-1,0) - release_count := co.dir.count(mask_incoherent) - crel_had_data := Bool(false) - } - state := Mux(_tag_match, - Mux(_needs_probes, s_probe, Mux(_is_hit, s_data_read, s_outer_read)), // Probe, hit or upgrade - Mux(_needs_writeback, s_wb_req, s_outer_read)) // Evict ifneedbe - } - } - is(s_wb_req) { - io.wb.req.valid := Bool(true) - when(io.wb.req.ready) { state := s_wb_resp } + is(s_outer_finish) { + io.outer.finish.valid := Bool(true) + when(io.outer.finish.ready) { state := s_wb_resp } } is(s_wb_resp) { - when(io.wb.resp.valid) { state := s_outer_read } - } - is(s_probe) { - // Send probes - io.inner.probe.valid := !co.dir.none(pending_probes) - when(io.inner.probe.ready) { - pending_probes := co.dir.pop(pending_probes, curr_p_id) - } - // Handle releases, which may have data being written back - //TODO: make sure cacq data is actually present before accpeting - // release data to merge! - io.inner.release.ready := Bool(true) - when(io.inner.release.valid) { - xact_meta.coh := next_coh_on_rel - // Handle released dirty data - when(crel.payload.hasData()) { - crel_had_data := Bool(true) - mergeData(xact_data, crel.payload) - } - // We don't decrement release_count until we've received all the data beats. - when(!crel.payload.hasMultibeatData() || crel_data_done) { - release_count := release_count - UInt(1) - } - } - when(release_count === UInt(0)) { - state := Mux(is_hit, Mux(crel_had_data, s_data_write, s_data_read), s_outer_read) - } - } - is(s_outer_read) { - io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_read - when(io.outer.acquire.ready) { - state := s_outer_resp - } - } - is(s_outer_resp) { - io.outer.grant.ready := Bool(true) - when(io.outer.grant.valid) { - //TODO make sure cacq data is actually present before merging - mergeData(xact_data, mgnt.payload) - when(mgnt_data_done) { - state := Mux(mgnt.payload.hasData(), s_data_write, s_data_read) - } - } - } - is(s_data_read) { - io.data.read.valid := !collect_cacq_data || cacq_data_valid(read_data_cnt) - when(io.data.resp.valid) { - mergeData(xact_data, io.data.resp.bits) - resp_data_cnt := resp_data_cnt + UInt(1) - } - when(io.data.read.ready) { - read_data_cnt := read_data_cnt + UInt(1) - when(read_data_cnt === read_data_max) { state := s_data_resp } - } - } - is(s_data_resp) { - when(io.data.resp.valid) { - mergeData(xact_data, io.data.resp.bits) - resp_data_cnt := resp_data_cnt + UInt(1) - } - when(resp_data_cnt === resp_data_max) { - state := Mux(xact.hasData(), s_data_write, s_meta_write) - } - } - is(s_data_write) { - io.data.write.valid := Bool(true) - when(io.data.write.ready) { - write_data_cnt := write_data_cnt + UInt(1) - when(write_data_cnt === write_data_max) { - state := s_meta_write - } - } - } - is(s_meta_write) { - io.meta.write.valid := Bool(true) - when(io.meta.write.ready) { state := s_grant } - } - is(s_grant) { - io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { - cgnt_data_cnt := cgnt_data_cnt + UInt(1) - when(cgnt_data_cnt === cgnt_data_max) { - state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) - } - } - } - is(s_ack) { - io.inner.finish.ready := Bool(true) - when(io.inner.finish.valid) { state := s_idle } + io.wb.resp.valid := Bool(true) + state := s_idle } } } diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 82f72dea..2524f3d1 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -3,202 +3,148 @@ package uncore import Chisel._ -// Classes to represent coherence information in clients and managers -abstract class CoherenceMetadata extends Bundle with CoherenceAgentParameters +/** The entire CoherencePolicy API consists of the following three traits: + * HasCustomTileLinkMessageTypes, used to define custom messages + * HasClientSideCoherencePolicy, for client coherence agents + * HasManagerSideCoherencePolicy, for manager coherence agents + */ +abstract class CoherencePolicy(val dir: DirectoryRepresentation) extends + HasCustomTileLinkMessageTypes with + HasClientSideCoherencePolicy with + HasManagerSideCoherencePolicy -class ClientMetadata extends CoherenceMetadata { - val state = UInt(width = co.clientStateWidth) - def ===(right: ClientMetadata): Bool = this.state === right.state - override def clone = new ClientMetadata().asInstanceOf[this.type] -} -object ClientMetadata { - def apply(state: UInt) = { - val meta = new ClientMetadata - meta.state := state - meta - } -} - -class ManagerMetadata extends CoherenceMetadata { - val state = UInt(width = co.masterStateWidth) - val sharers = UInt(width = co.dir.width) - override def clone = new ManagerMetadata().asInstanceOf[this.type] -} -object ManagerMetadata { - def apply(state: UInt, sharers: UInt) = { - val meta = new ManagerMetadata - meta.state := state - meta.sharers := sharers - meta - } - def apply(state: UInt): ManagerMetadata = apply(state, new ManagerMetadata().co.dir.flush) -} - -// This class encapsulates transformations on different directory information -// storage formats -abstract class DirectoryRepresentation(val width: Int) { - def pop(prev: UInt, id: UInt): UInt - def push(prev: UInt, id: UInt): UInt - def flush: UInt - def none(s: UInt): Bool - def one(s: UInt): Bool - def count(s: UInt): UInt - def next(s: UInt): UInt - def full(s: UInt): UInt -} - -class NullRepresentation(nClients: Int) extends DirectoryRepresentation(1) { - def pop(prev: UInt, id: UInt) = UInt(0) - def push(prev: UInt, id: UInt) = UInt(0) - def flush = UInt(0) - def none(s: UInt) = Bool(false) - def one(s: UInt) = Bool(false) - def count(s: UInt) = UInt(nClients) - def next(s: UInt) = UInt(0) - def full(s: UInt) = SInt(-1, width = nClients).toUInt -} - -class FullRepresentation(nClients: Int) extends DirectoryRepresentation(nClients) { - def pop(prev: UInt, id: UInt) = prev & ~UIntToOH(id) - def push(prev: UInt, id: UInt) = prev | UIntToOH(id) - def flush = UInt(0, width = width) - def none(s: UInt) = s === UInt(0) - def one(s: UInt) = PopCount(s) === UInt(1) - def count(s: UInt) = PopCount(s) - def next(s: UInt) = PriorityEncoder(s) - def full(s: UInt) = s -} - -// Coherence policy inferface for clients and managers -abstract class CoherencePolicy(val dir: DirectoryRepresentation) { - def nClientStates: Int - def nManagerStates: Int - def nAcquireTypes: Int - def nProbeTypes: Int - def nReleaseTypes: Int - def nGrantTypes: Int - def clientStateWidth = log2Up(nClientStates) - def masterStateWidth = log2Up(nManagerStates) +/** This API defines the custom, coherence-policy-defined message types, + * as opposed to the built-in ones found in tilelink.scala. + * Policies must enumerate the custom messages to be sent over each + * channel, as well as which of them have associated data. + */ +trait HasCustomTileLinkMessageTypes { + val nAcquireTypes: Int def acquireTypeWidth = log2Up(nAcquireTypes) + val nProbeTypes: Int def probeTypeWidth = log2Up(nProbeTypes) + val nReleaseTypes: Int def releaseTypeWidth = log2Up(nReleaseTypes) + val nGrantTypes: Int def grantTypeWidth = log2Up(nGrantTypes) - val clientStatesWithReadPermission: Vec[UInt] - val clientStatesWithWritePermission: Vec[UInt] - val clientStatesWithDirtyData: Vec[UInt] val acquireTypesWithData = Nil // Only built-in Acquire types have data for now val releaseTypesWithData: Vec[UInt] val grantTypesWithData: Vec[UInt] +} +/** This API contains all functions required for client coherence agents. + * Policies must enumerate the number of client states and define their + * permissions with respect to memory operations. Policies must fill in functions + * to control which messages are sent and how metadata is updated in response + * to coherence events. These funtions are generally called from within the + * ClientMetadata class in metadata.scala + */ +trait HasClientSideCoherencePolicy { + // Client coherence states and their permissions + val nClientStates: Int + def clientStateWidth = log2Ceil(nClientStates) + val clientStatesWithReadPermission: Vec[UInt] + val clientStatesWithWritePermission: Vec[UInt] + val clientStatesWithDirtyData: Vec[UInt] + + // Transaction initiation logic def isValid(meta: ClientMetadata): Bool - def isValid(meta: ManagerMetadata): Bool - def isHit(cmd: UInt, meta: ClientMetadata): Bool = { Mux(isWriteIntent(cmd), clientStatesWithWritePermission.contains(meta.state), clientStatesWithReadPermission.contains(meta.state)) } - //TODO: Use outer protocol's clientState instead, remove this function: - def isHit(incoming: Acquire, meta: ManagerMetadata) = isValid(meta) - - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool + //TODO: Assumes all states with write permissions also have read permissions + def requiresAcquireOnSecondaryMiss( + first_cmd: UInt, + second_cmd: UInt, + meta: ClientMetadata): Bool = { + isWriteIntent(second_cmd) && + !isWriteIntent(first_cmd) || + (getAcquireType(first_cmd, meta) != getAcquireType(second_cmd, meta)) + } //TODO: Assumes all cache ctrl ops writeback dirty data, and // doesn't issue transaction when e.g. downgrading Exclusive to Shared: - def needsTransactionOnCacheControl(cmd: UInt, meta: ClientMetadata): Bool = + def requiresReleaseOnCacheControl(cmd: UInt, meta: ClientMetadata): Bool = clientStatesWithDirtyData.contains(meta.state) - def needsWriteback(meta: ClientMetadata): Bool = - needsTransactionOnCacheControl(M_FLUSH, meta) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire): ClientMetadata - def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata): ClientMetadata + // Determine which custom message type to use + def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt + def getReleaseType(p: Probe, meta: ClientMetadata): UInt + + // Mutate ClientMetadata based on messages or cmds + def clientMetadataOnReset: ClientMetadata def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata): ClientMetadata def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata): ClientMetadata - def clientMetadataOnFlush: ClientMetadata - - def managerMetadataOnRelease(incoming: Release, meta: ManagerMetadata, src: UInt): ManagerMetadata - def managerMetadataOnGrant(outgoing: Grant, meta: ManagerMetadata, dst: UInt): ManagerMetadata - def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata): ManagerMetadata - def managerMetadataOnFlush: ManagerMetadata - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt - def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt - def getReleaseType(p: Probe, meta: ClientMetadata): UInt - def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt - - def getGrantType(acq: Acquire, meta: ManagerMetadata): UInt - def getProbeType(acq: Acquire, meta: ManagerMetadata): UInt - def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt - - def requiresOuterRead(acq: Acquire, meta: ManagerMetadata): Bool - def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata): Bool - def requiresProbes(acq: Acquire, meta: ManagerMetadata): Bool - def requiresProbes(cmd: UInt, meta: ManagerMetadata): Bool - def requiresProbesOnVoluntaryWriteback(meta: ManagerMetadata): Bool = requiresProbes(M_FLUSH, meta) + def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata): ClientMetadata + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata): ClientMetadata } -class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { - def nClientStates = 2 - def nManagerStates = 2 - def nAcquireTypes = 1 - def nProbeTypes = 2 - def nReleaseTypes = 4 - def nGrantTypes = 1 +/** This API contains all functions required for manager coherence agents. + * Policies must enumerate the number of manager states. Policies must fill + * in functions to control which Probe and Grant messages are sent and how + * metadata should be updated in response to coherence events. These funtions + * are generally called from within the ManagerMetadata class in metadata.scala + */ +trait HasManagerSideCoherencePolicy extends HasDirectoryRepresentation { + val nManagerStates: Int + def masterStateWidth = log2Ceil(nManagerStates) - val clientInvalid :: clientValid :: Nil = Enum(UInt(), nClientStates) - val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) + // Transaction probing logic + def requiresProbes(acq: Acquire, meta: ManagerMetadata): Bool + def requiresProbes(cmd: UInt, meta: ManagerMetadata): Bool + + // Determine which custom message type to use in response + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt + def getProbeType(acq: Acquire, meta: ManagerMetadata): UInt + def getGrantType(acq: Acquire, meta: ManagerMetadata): UInt + + // Mutate ManagerMetadata based on messages or cmds + def managerMetadataOnReset: ManagerMetadata + def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata): ManagerMetadata + def managerMetadataOnGrant(outgoing: Grant, dst: UInt, meta: ManagerMetadata) = + ManagerMetadata(sharers=Mux(outgoing.isBuiltInType(), // Assumes all built-ins are uncached + meta.sharers, + dir.push(meta.sharers, dst))) + //state = meta.state) TODO: Fix 0-width wires in Chisel +} + +/** The following concrete implementations of CoherencePolicy each provide the + * functionality of one particular protocol. + */ + +/** A simple protocol with only two Client states. + * Data is always assumed to be dirty. + * Only a single client may ever have a copy of a block at a time. + */ +class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { + // Message types + val nAcquireTypes = 1 + val nProbeTypes = 2 + val nReleaseTypes = 4 + val nGrantTypes = 1 val acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) val releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) val grantExclusive :: Nil = Enum(UInt(), nGrantTypes) - val clientStatesWithReadPermission = Vec(clientValid) - val clientStatesWithWritePermission = Vec(clientValid) - val clientStatesWithDirtyData = Vec(clientValid) val releaseTypesWithData = Vec(releaseInvalidateData, releaseCopyData) val grantTypesWithData = Vec(grantExclusive) + // Client states and functions + val nClientStates = 2 + val clientInvalid :: clientValid :: Nil = Enum(UInt(), nClientStates) + + val clientStatesWithReadPermission = Vec(clientValid) + val clientStatesWithWritePermission = Vec(clientValid) + val clientStatesWithDirtyData = Vec(clientValid) + def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid - def isValid (meta: ManagerMetadata): Bool = meta.state != managerInvalid - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = - (outstanding.a_type != acquireExclusive) - - def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = meta - - def clientMetadataOnFlush = ClientMetadata(clientInvalid) - def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = - ClientMetadata(Mux(cmd === M_FLUSH, clientInvalid, meta.state)) - - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = - ClientMetadata(Mux(incoming.builtin_type, clientInvalid, clientValid)) - - def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = - ClientMetadata(Mux(incoming.p_type === probeInvalidate, - clientInvalid, meta.state)) - - def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(managerValid, dir.pop(meta.sharers, src)) - MuxBundle(meta, Array( - r.is(releaseInvalidateData) -> next, - r.is(releaseInvalidateAck) -> next - )) - } - - def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.builtin_type, - ManagerMetadata(managerValid, meta.sharers), - ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) - - def managerMetadataOnFlush = ManagerMetadata(managerInvalid) - - def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = - ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = acquireExclusive - - def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = acquireExclusive + def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = acquireExclusive def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { val dirty = clientStatesWithDirtyData.contains(meta.state) @@ -213,112 +159,85 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeInvalidate -> getReleaseType(M_FLUSH, meta), probeCopy -> getReleaseType(M_CLEAN, meta))) - def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) + def clientMetadataOnReset = ClientMetadata(clientInvalid) - def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), grantExclusive) + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = meta - def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, - MuxLookup(a.a_type, probeCopy, Array( - Acquire.uncachedReadBlock -> probeCopy, - Acquire.uncachedWriteBlock -> probeInvalidate, - Acquire.uncachedRead -> probeCopy, - Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate)), - probeInvalidate) + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(cmd === M_FLUSH, clientInvalid, meta.state)) + + def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(incoming.isBuiltInType(), clientInvalid, clientValid)) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata(Mux(incoming.p_type === probeInvalidate, + clientInvalid, meta.state)) + + // Manager states and functions: + val nManagerStates = 0 // We don't actually need any states for this protocol + + def requiresProbes(a: Acquire, meta: ManagerMetadata) = !dir.none(meta.sharers) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = MuxLookup(cmd, probeCopy, Array( M_FLUSH -> probeInvalidate)) - def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.isBuiltInType(), + MuxLookup(a.a_type, probeCopy, Array( + Acquire.getBlockType -> probeCopy, + Acquire.putBlockType -> probeInvalidate, + Acquire.getType -> probeCopy, + Acquire.putType -> probeInvalidate, + Acquire.putAtomicType -> probeInvalidate)), + probeInvalidate) - def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = grantExclusive - def requiresProbes(a: Acquire, meta: ManagerMetadata) = !dir.none(meta.sharers) + def managerMetadataOnReset = ManagerMetadata() - def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) + def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { + val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src)) + MuxBundle(meta, Array( + incoming.is(releaseInvalidateData) -> popped, + incoming.is(releaseInvalidateAck) -> popped)) + } } +/** A simple protocol with only three Client states. + * Data is marked as dirty when written. + * Only a single client may ever have a copy of a block at a time. + */ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { - def nClientStates = 3 - def nManagerStates = 2 - def nAcquireTypes = 2 - def nProbeTypes = 3 - def nReleaseTypes = 6 - def nGrantTypes = 1 - - val clientInvalid :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) + // Message types + val nAcquireTypes = 2 + val nProbeTypes = 3 + val nReleaseTypes = 6 + val nGrantTypes = 1 val acquireExclusiveClean :: acquireExclusiveDirty :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) val grantExclusive :: Nil = Enum(UInt(), nGrantTypes) - val clientStatesWithReadPermission = Vec(clientExclusiveClean, clientExclusiveDirty) - val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) - val clientStatesWithDirtyData = Vec(clientExclusiveDirty) val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) val grantTypesWithData = Vec(grantExclusive) + // Client states and functions + val nClientStates = 3 + val clientInvalid :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) + + val clientStatesWithReadPermission = Vec(clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty) + def isValid (meta: ClientMetadata) = meta.state != clientInvalid - def isValid (meta: ManagerMetadata) = meta.state != managerInvalid - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = - (isRead(cmd) && outstanding.builtin_type) || - (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusiveDirty)) - - def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = - ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) - - def clientMetadataOnFlush = ClientMetadata(clientInvalid) - def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = - ClientMetadata( - MuxLookup(cmd, meta.state, Array( - M_FLUSH -> clientInvalid, - M_CLEAN -> Mux(meta.state === clientExclusiveDirty, clientExclusiveClean, meta.state)))) - - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = - ClientMetadata( - Mux(incoming.builtin_type, clientInvalid, - Mux(outstanding.a_type === acquireExclusiveDirty, clientExclusiveDirty, - clientExclusiveClean))) - - def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = - ClientMetadata( - MuxLookup(incoming.p_type, meta.state, Array( - probeInvalidate -> clientInvalid, - probeDowngrade -> clientExclusiveClean, - probeCopy -> meta.state))) - - def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(managerValid, dir.pop(meta.sharers,src)) - MuxBundle(meta, Array( - r.is(releaseInvalidateData) -> next, - r.is(releaseInvalidateAck) -> next - )) - } - - def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.builtin_type, - ManagerMetadata(managerValid, meta.sharers), - ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) - - def managerMetadataOnFlush = ManagerMetadata(managerInvalid) - - def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = - ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = + def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = Mux(isWriteIntent(cmd), acquireExclusiveDirty, acquireExclusiveClean) - def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = - Mux(isWriteIntent(cmd), acquireExclusiveDirty, outstanding.a_type) - def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { val dirty = clientStatesWithDirtyData.contains(meta.state) MuxLookup(cmd, releaseCopyAck, Array( @@ -333,75 +252,113 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeDowngrade -> getReleaseType(M_PRODUCE, meta), probeCopy -> getReleaseType(M_CLEAN, meta))) - def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) + def clientMetadataOnReset = ClientMetadata(clientInvalid) - def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), grantExclusive) + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) - def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, - MuxLookup(a.a_type, probeCopy, Array( - Acquire.uncachedReadBlock -> probeCopy, - Acquire.uncachedWriteBlock -> probeInvalidate, - Acquire.uncachedRead -> probeCopy, - Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate)), - probeInvalidate) + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(cmd, meta.state, Array( + M_FLUSH -> clientInvalid, + M_CLEAN -> Mux(meta.state === clientExclusiveDirty, clientExclusiveClean, meta.state)))) + + def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + Mux(incoming.isBuiltInType(), clientInvalid, + Mux(isWrite(cmd), clientExclusiveDirty, clientExclusiveClean))) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(incoming.p_type, meta.state, Array( + probeInvalidate -> clientInvalid, + probeDowngrade -> clientExclusiveClean, + probeCopy -> meta.state))) + + // Manager states and functions: + val nManagerStates = 0 // We don't actually need any states for this protocol + + def requiresProbes(a: Acquire, meta: ManagerMetadata) = !dir.none(meta.sharers) + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = MuxLookup(cmd, probeCopy, Array( M_FLUSH -> probeInvalidate, M_PRODUCE -> probeDowngrade)) - def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.isBuiltInType(), + MuxLookup(a.a_type, probeCopy, Array( + Acquire.getBlockType -> probeCopy, + Acquire.putBlockType -> probeInvalidate, + Acquire.getType -> probeCopy, + Acquire.putType -> probeInvalidate, + Acquire.putAtomicType -> probeInvalidate)), + probeInvalidate) - def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = grantExclusive - def requiresProbes(a: Acquire, meta: ManagerMetadata) = - Mux(dir.none(meta.sharers), Bool(false), - Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive - Mux(a.builtin_type, a.hasData(), Bool(true)))) + def managerMetadataOnReset = ManagerMetadata() - def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) + def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { + val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src)) + MuxBundle(meta, Array( + incoming.is(releaseInvalidateData) -> popped, + incoming.is(releaseInvalidateAck) -> popped)) + } } +/** A protocol with only three Client states. + * Data is always assumed to be dirty. + * Multiple clients may share read permissions on a block at the same time. + */ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { - def nClientStates = 3 - def nManagerStates = 2 - def nAcquireTypes = 2 - def nProbeTypes = 3 - def nReleaseTypes = 6 - def nGrantTypes = 3 - - val clientInvalid :: clientShared :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - //val managerInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) - val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) + // Message types + val nAcquireTypes = 2 + val nProbeTypes = 3 + val nReleaseTypes = 6 + val nGrantTypes = 3 val acquireShared :: acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) val grantShared :: grantExclusive :: grantExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) - val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveDirty) - val clientStatesWithWritePermission = Vec(clientExclusiveDirty) - val clientStatesWithDirtyData = Vec(clientExclusiveDirty) val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) val grantTypesWithData = Vec(grantShared, grantExclusive) - def isValid(meta: ClientMetadata): Bool = meta.state != clientInvalid - def isValid(meta: ManagerMetadata) = meta.state != managerInvalid + // Client states and functions + val nClientStates = 3 + val clientInvalid :: clientShared :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = - (isRead(cmd) && outstanding.builtin_type) || - (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusive)) + val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty) + + def isValid(meta: ClientMetadata): Bool = meta.state != clientInvalid + + def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = + Mux(isWriteIntent(cmd), acquireExclusive, acquireShared) + + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) + MuxLookup(cmd, releaseCopyAck, Array( + M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), + M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), + M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) + } + + def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = + MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> getReleaseType(M_FLUSH, meta), + probeDowngrade -> getReleaseType(M_PRODUCE, meta), + probeCopy -> getReleaseType(M_CLEAN, meta))) + + def clientMetadataOnReset = ClientMetadata(clientInvalid) def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) - def clientMetadataOnFlush = ClientMetadata(clientInvalid) - def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = ClientMetadata( MuxLookup(cmd, meta.state, Array( @@ -409,9 +366,9 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), clientShared, meta.state)))) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = + def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = ClientMetadata( - Mux(incoming.builtin_type, clientInvalid, + Mux(incoming.isBuiltInType(), clientInvalid, MuxLookup(incoming.g_type, clientInvalid, Array( grantShared -> clientShared, grantExclusive -> clientExclusiveDirty, @@ -424,30 +381,82 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeDowngrade -> clientShared, probeCopy -> meta.state))) - def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(managerValid, dir.pop(meta.sharers,src)) + // Manager states and functions: + val nManagerStates = 0 // TODO: We could add a Shared state to avoid probing + // only a single sharer (also would need + // notification msg to track clean drops) + + def requiresProbes(a: Acquire, meta: ManagerMetadata) = + Mux(dir.none(meta.sharers), Bool(false), + Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive + Mux(a.isBuiltInType(), a.hasData(), a.a_type != acquireShared))) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) + + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = + MuxLookup(cmd, probeCopy, Array( + M_FLUSH -> probeInvalidate, + M_PRODUCE -> probeDowngrade)) + + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.isBuiltInType(), + MuxLookup(a.a_type, probeCopy, Array( + Acquire.getBlockType -> probeCopy, + Acquire.putBlockType -> probeInvalidate, + Acquire.getType -> probeCopy, + Acquire.putType -> probeInvalidate, + Acquire.putAtomicType -> probeInvalidate)), + MuxLookup(a.a_type, probeCopy, Array( + acquireShared -> probeDowngrade, + acquireExclusive -> probeInvalidate))) + + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.a_type === acquireShared, + Mux(!dir.none(meta.sharers), grantShared, grantExclusive), + grantExclusive) + + def managerMetadataOnReset = ManagerMetadata() + + def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { + val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src)) MuxBundle(meta, Array( - r.is(releaseInvalidateData) -> next, - r.is(releaseInvalidateAck) -> next - )) + incoming.is(releaseInvalidateData) -> popped, + incoming.is(releaseInvalidateAck) -> popped)) } +} - def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.builtin_type, - ManagerMetadata(managerValid, meta.sharers), - ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) +/** A protocol with four Client states. + * Data is marked as dirty when written. + * Multiple clients may share read permissions on a block at the same time. + */ +class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { + // Message types + val nAcquireTypes = 2 + val nProbeTypes = 3 + val nReleaseTypes = 6 + val nGrantTypes = 3 - def managerMetadataOnFlush = ManagerMetadata(managerInvalid) + val acquireShared :: acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantShared :: grantExclusive :: grantExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) - def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = - ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) + val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val grantTypesWithData = Vec(grantShared, grantExclusive) - def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = + // Client states and functions + val nClientStates = 4 + val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) + + val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty) + + def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid + + def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = Mux(isWriteIntent(cmd), acquireExclusive, acquireShared) - def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = - Mux(isWriteIntent(cmd), acquireExclusive, outstanding.a_type) - def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { val dirty = clientStatesWithDirtyData.contains(meta.state) MuxLookup(cmd, releaseCopyAck, Array( @@ -462,80 +471,11 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeDowngrade -> getReleaseType(M_PRODUCE, meta), probeCopy -> getReleaseType(M_CLEAN, meta))) - def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - - def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), - Mux(a.a_type === acquireShared, - Mux(!dir.none(meta.sharers), grantShared, grantExclusive), - grantExclusive)) - - def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, - MuxLookup(a.a_type, probeCopy, Array( - Acquire.uncachedReadBlock -> probeCopy, - Acquire.uncachedWriteBlock -> probeInvalidate, - Acquire.uncachedRead -> probeCopy, - Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate)), - MuxLookup(a.a_type, probeCopy, Array( - acquireShared -> probeDowngrade, - acquireExclusive -> probeInvalidate))) - - def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = - MuxLookup(cmd, probeCopy, Array( - M_FLUSH -> probeInvalidate, - M_PRODUCE -> probeDowngrade)) - - def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), Bool(true)) - - def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) - - def requiresProbes(a: Acquire, meta: ManagerMetadata) = - Mux(dir.none(meta.sharers), Bool(false), - Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive - Mux(a.builtin_type, a.hasData(), a.a_type != acquireShared))) - - def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) -} - -class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { - def nClientStates = 4 - def nManagerStates = 2 - def nAcquireTypes = 2 - def nProbeTypes = 3 - def nReleaseTypes = 6 - def nGrantTypes = 3 - - val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - //val managerInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) - val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) - - val acquireShared :: acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) - val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantShared :: grantExclusive :: grantExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) - - val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty) - val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) - val clientStatesWithDirtyData = Vec(clientExclusiveDirty) - val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val grantTypesWithData = Vec(grantShared, grantExclusive) - - def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid - def isValid (meta: ManagerMetadata) = meta.state != managerInvalid - - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = - (isRead(cmd) && outstanding.builtin_type) || - (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusive)) + def clientMetadataOnReset = ClientMetadata(clientInvalid) def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) - def clientMetadataOnFlush = ClientMetadata(clientInvalid) - def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = ClientMetadata( MuxLookup(cmd, meta.state, Array( @@ -544,13 +484,12 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { clientShared, meta.state), M_CLEAN -> Mux(meta.state === clientExclusiveDirty, clientExclusiveClean, meta.state)))) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = + def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = ClientMetadata( - Mux(incoming.builtin_type, clientInvalid, + Mux(incoming.isBuiltInType(), clientInvalid, MuxLookup(incoming.g_type, clientInvalid, Array( grantShared -> clientShared, - grantExclusive -> Mux(outstanding.a_type === acquireExclusive, - clientExclusiveDirty, clientExclusiveClean), + grantExclusive -> Mux(isWrite(cmd), clientExclusiveDirty, clientExclusiveClean), grantExclusiveAck -> clientExclusiveDirty)))) def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = @@ -560,188 +499,80 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeDowngrade -> clientShared, probeCopy -> meta.state))) - def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(managerValid, dir.pop(meta.sharers,src)) - MuxBundle(meta, Array( - r.is(releaseInvalidateData) -> next, - r.is(releaseInvalidateAck) -> next - )) - } + // Manager states and functions: + val nManagerStates = 0 // TODO: We could add a Shared state to avoid probing + // only a single sharer (also would need + // notification msg to track clean drops) - def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.builtin_type, - ManagerMetadata(managerValid, meta.sharers), - ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) + def requiresProbes(a: Acquire, meta: ManagerMetadata) = + Mux(dir.none(meta.sharers), Bool(false), + Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive + Mux(a.isBuiltInType(), a.hasData(), a.a_type != acquireShared))) - def managerMetadataOnFlush = ManagerMetadata(managerInvalid) - - def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = - ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = - Mux(isWriteIntent(cmd), acquireExclusive, acquireShared) - - def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = - Mux(isWriteIntent(cmd), acquireExclusive, outstanding.a_type) - - def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { - val dirty = clientStatesWithDirtyData.contains(meta.state) - MuxLookup(cmd, releaseCopyAck, Array( - M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), - M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), - M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) - } - - def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = - MuxLookup(incoming.p_type, releaseInvalidateAck, Array( - probeInvalidate -> getReleaseType(M_FLUSH, meta), - probeDowngrade -> getReleaseType(M_PRODUCE, meta), - probeCopy -> getReleaseType(M_CLEAN, meta))) - - def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - - def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), - Mux(a.a_type === acquireShared, - Mux(!dir.none(meta.sharers), grantShared, grantExclusive), - grantExclusive)) - - def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, - MuxLookup(a.a_type, probeCopy, Array( - Acquire.uncachedReadBlock -> probeCopy, - Acquire.uncachedWriteBlock -> probeInvalidate, - Acquire.uncachedRead -> probeCopy, - Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate)), - MuxLookup(a.a_type, probeCopy, Array( - acquireShared -> probeDowngrade, - acquireExclusive -> probeInvalidate))) + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = MuxLookup(cmd, probeCopy, Array( M_FLUSH -> probeInvalidate, M_PRODUCE -> probeDowngrade)) - def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.isBuiltInType(), + MuxLookup(a.a_type, probeCopy, Array( + Acquire.getBlockType -> probeCopy, + Acquire.putBlockType -> probeInvalidate, + Acquire.getType -> probeCopy, + Acquire.putType -> probeInvalidate, + Acquire.putAtomicType -> probeInvalidate)), + MuxLookup(a.a_type, probeCopy, Array( + acquireShared -> probeDowngrade, + acquireExclusive -> probeInvalidate))) - def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.a_type === acquireShared, + Mux(!dir.none(meta.sharers), grantShared, grantExclusive), + grantExclusive) - def requiresProbes(a: Acquire, meta: ManagerMetadata) = - Mux(dir.none(meta.sharers), Bool(false), - Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive - Mux(a.builtin_type, a.hasData(), a.a_type != acquireShared))) + def managerMetadataOnReset = ManagerMetadata() - def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) + def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { + val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src)) + MuxBundle(meta, Array( + incoming.is(releaseInvalidateData) -> popped, + incoming.is(releaseInvalidateAck) -> popped)) + } } class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { - def nClientStates = 7 - def nManagerStates = 2 - def nAcquireTypes = 3 - def nProbeTypes = 4 - def nReleaseTypes = 10 - def nGrantTypes = 4 - - val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: clientSharedByTwo :: clientMigratoryClean :: clientMigratoryDirty :: Nil = Enum(UInt(), nClientStates) - //val managerInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) - val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) + // Message types + val nAcquireTypes = 3 + val nProbeTypes = 4 + val nReleaseTypes = 10 + val nGrantTypes = 4 val acquireShared :: acquireExclusive :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(UInt(), nProbeTypes) val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(UInt(), nReleaseTypes) val grantShared :: grantExclusive :: grantExclusiveAck :: grantReadMigratory :: Nil = Enum(UInt(), nGrantTypes) - val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty, clientSharedByTwo, clientMigratoryClean, clientMigratoryDirty) - val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty, clientMigratoryClean, clientMigratoryDirty) - val clientStatesWithDirtyData = Vec(clientExclusiveDirty, clientMigratoryDirty) val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) val grantTypesWithData = Vec(grantShared, grantExclusive, grantReadMigratory) + // Client states and functions + val nClientStates = 7 + val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: clientSharedByTwo :: clientMigratoryClean :: clientMigratoryDirty :: Nil = Enum(UInt(), nClientStates) + + val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty, clientSharedByTwo, clientMigratoryClean, clientMigratoryDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty, clientMigratoryClean, clientMigratoryDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty, clientMigratoryDirty) + def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid - def isValid (meta: ManagerMetadata) = meta.state != managerInvalid - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = - (isRead(cmd) && outstanding.builtin_type) || - (isWriteIntent(cmd) && !Vec(acquireExclusive, acquireInvalidateOthers).contains(outstanding.a_type)) - - def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = - ClientMetadata( - Mux(isWrite(cmd), MuxLookup(meta.state, clientExclusiveDirty, Array( - clientExclusiveClean -> clientExclusiveDirty, - clientMigratoryClean -> clientMigratoryDirty)), - meta.state)) - - def clientMetadataOnFlush = ClientMetadata(clientInvalid) - - def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = - ClientMetadata( - MuxLookup(cmd, meta.state, Array( - M_FLUSH -> clientInvalid, - M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), - clientShared, meta.state), - M_CLEAN -> MuxLookup(meta.state, meta.state, Array( - clientExclusiveDirty -> clientExclusiveClean, - clientMigratoryDirty -> clientMigratoryClean))))) - - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = - ClientMetadata( - Mux(incoming.builtin_type, clientInvalid, - MuxLookup(incoming.g_type, clientInvalid, Array( - grantShared -> clientShared, - grantExclusive -> MuxLookup(outstanding.a_type, clientExclusiveDirty, Array( - acquireExclusive -> clientExclusiveDirty, - acquireShared -> clientExclusiveClean)), - grantExclusiveAck -> clientExclusiveDirty, - grantReadMigratory -> MuxLookup(outstanding.a_type, clientMigratoryDirty, Array( - acquireInvalidateOthers -> clientMigratoryDirty, - acquireExclusive -> clientMigratoryDirty, - acquireShared -> clientMigratoryClean)))))) - - def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = - ClientMetadata( - MuxLookup(incoming.p_type, meta.state, Array( - probeInvalidate -> clientInvalid, - probeInvalidateOthers -> clientInvalid, - probeCopy -> meta.state, - probeDowngrade -> MuxLookup(meta.state, clientShared, Array( - clientExclusiveClean -> clientSharedByTwo, - clientExclusiveDirty -> clientSharedByTwo, - clientSharedByTwo -> clientShared, - clientMigratoryClean -> clientSharedByTwo, - clientMigratoryDirty -> clientInvalid))))) - - def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(managerValid, dir.pop(meta.sharers,src)) - MuxBundle(meta, Array( - r.is(releaseInvalidateData) -> next, - r.is(releaseInvalidateAck) -> next, - r.is(releaseInvalidateDataMigratory) -> next, - r.is(releaseInvalidateAckMigratory) -> next)) - } - - def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.builtin_type, - ManagerMetadata(managerValid, meta.sharers), - ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) - - def managerMetadataOnFlush = ManagerMetadata(managerInvalid) - - def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = - ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = + def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = Mux(isWriteIntent(cmd), Mux(meta.state === clientInvalid, acquireExclusive, acquireInvalidateOthers), acquireShared) - def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = - Mux(isWriteIntent(cmd), - Mux(meta.state === clientInvalid, acquireExclusive, acquireInvalidateOthers), - outstanding.a_type) - def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { val dirty = clientStatesWithDirtyData.contains(meta.state) MuxLookup(cmd, releaseCopyAck, Array( @@ -769,43 +600,89 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d Mux(dirty, with_data, without_data) } - def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) + def clientMetadataOnReset = ClientMetadata(clientInvalid) - def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), - MuxLookup(a.a_type, grantShared, Array( - acquireShared -> Mux(!dir.none(meta.sharers), grantShared, grantExclusive), - acquireExclusive -> grantExclusive, - acquireInvalidateOthers -> grantExclusiveAck))) //TODO: add this to MESI for broadcast? + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + Mux(isWrite(cmd), MuxLookup(meta.state, clientExclusiveDirty, Array( + clientExclusiveClean -> clientExclusiveDirty, + clientMigratoryClean -> clientMigratoryDirty)), + meta.state)) - def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, - MuxLookup(a.a_type, probeCopy, Array( - Acquire.uncachedReadBlock -> probeCopy, - Acquire.uncachedWriteBlock -> probeInvalidate, - Acquire.uncachedRead -> probeCopy, - Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate)), - MuxLookup(a.a_type, probeCopy, Array( - acquireShared -> probeDowngrade, - acquireExclusive -> probeInvalidate, - acquireInvalidateOthers -> probeInvalidateOthers))) + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(cmd, meta.state, Array( + M_FLUSH -> clientInvalid, + M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), + clientShared, meta.state), + M_CLEAN -> MuxLookup(meta.state, meta.state, Array( + clientExclusiveDirty -> clientExclusiveClean, + clientMigratoryDirty -> clientMigratoryClean))))) + + def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + Mux(incoming.isBuiltInType(), clientInvalid, + MuxLookup(incoming.g_type, clientInvalid, Array( + grantShared -> clientShared, + grantExclusive -> Mux(isWrite(cmd), clientExclusiveDirty, clientExclusiveClean), + grantExclusiveAck -> clientExclusiveDirty, + grantReadMigratory -> Mux(isWrite(cmd), clientMigratoryDirty, clientMigratoryClean))))) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(incoming.p_type, meta.state, Array( + probeInvalidate -> clientInvalid, + probeInvalidateOthers -> clientInvalid, + probeCopy -> meta.state, + probeDowngrade -> MuxLookup(meta.state, clientShared, Array( + clientExclusiveClean -> clientSharedByTwo, + clientExclusiveDirty -> clientSharedByTwo, + clientSharedByTwo -> clientShared, + clientMigratoryClean -> clientSharedByTwo, + clientMigratoryDirty -> clientInvalid))))) + + // Manager states and functions: + val nManagerStates = 0 // TODO: we could add some states to reduce the number of message types + + def requiresProbes(a: Acquire, meta: ManagerMetadata) = + Mux(dir.none(meta.sharers), Bool(false), + Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive + Mux(a.isBuiltInType(), a.hasData(), a.a_type != acquireShared))) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = MuxLookup(cmd, probeCopy, Array( M_FLUSH -> probeInvalidate, M_PRODUCE -> probeDowngrade)) - def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), acq.a_type != acquireInvalidateOthers) + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.isBuiltInType(), + MuxLookup(a.a_type, probeCopy, Array( + Acquire.getBlockType -> probeCopy, + Acquire.putBlockType -> probeInvalidate, + Acquire.getType -> probeCopy, + Acquire.putType -> probeInvalidate, + Acquire.putAtomicType -> probeInvalidate)), + MuxLookup(a.a_type, probeCopy, Array( + acquireShared -> probeDowngrade, + acquireExclusive -> probeInvalidate, + acquireInvalidateOthers -> probeInvalidateOthers))) - def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + MuxLookup(a.a_type, grantShared, Array( + acquireShared -> Mux(!dir.none(meta.sharers), grantShared, grantExclusive), + acquireExclusive -> grantExclusive, + acquireInvalidateOthers -> grantExclusiveAck)) //TODO: add this to MESI for broadcast? - def requiresProbes(a: Acquire, meta: ManagerMetadata) = - Mux(dir.none(meta.sharers), Bool(false), - Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive - Mux(a.builtin_type, a.hasData(), a.a_type != acquireShared))) + def managerMetadataOnReset = ManagerMetadata() - def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) + def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { + val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src)) + MuxBundle(meta, Array( + incoming.is(releaseInvalidateData) -> popped, + incoming.is(releaseInvalidateAck) -> popped, + incoming.is(releaseInvalidateDataMigratory) -> popped, + incoming.is(releaseInvalidateAckMigratory) -> popped)) + } } diff --git a/uncore/src/main/scala/directory.scala b/uncore/src/main/scala/directory.scala new file mode 100644 index 00000000..db555ad3 --- /dev/null +++ b/uncore/src/main/scala/directory.scala @@ -0,0 +1,43 @@ +// See LICENSE for license details. + +package uncore +import Chisel._ + +// This class encapsulates transformations on different directory information +// storage formats +abstract class DirectoryRepresentation(val width: Int) { + def pop(prev: UInt, id: UInt): UInt + def push(prev: UInt, id: UInt): UInt + def flush: UInt + def none(s: UInt): Bool + def one(s: UInt): Bool + def count(s: UInt): UInt + def next(s: UInt): UInt + def full(s: UInt): UInt +} + +abstract trait HasDirectoryRepresentation { + val dir: DirectoryRepresentation +} + +class NullRepresentation(nClients: Int) extends DirectoryRepresentation(1) { + def pop(prev: UInt, id: UInt) = UInt(0) + def push(prev: UInt, id: UInt) = UInt(0) + def flush = UInt(0) + def none(s: UInt) = Bool(false) + def one(s: UInt) = Bool(false) + def count(s: UInt) = UInt(nClients) + def next(s: UInt) = UInt(0) + def full(s: UInt) = SInt(-1, width = nClients).toUInt +} + +class FullRepresentation(nClients: Int) extends DirectoryRepresentation(nClients) { + def pop(prev: UInt, id: UInt) = prev & ~UIntToOH(id) + def push(prev: UInt, id: UInt) = prev | UIntToOH(id) + def flush = UInt(0, width = width) + def none(s: UInt) = s === UInt(0) + def one(s: UInt) = PopCount(s) === UInt(1) + def count(s: UInt) = PopCount(s) + def next(s: UInt) = PriorityEncoder(s) + def full(s: UInt) = s +} diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 5e5053be..4d5e1135 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -14,7 +14,6 @@ case object HTIFNCores extends Field[Int] abstract trait HTIFParameters extends UsesParameters { val dataBits = params(TLDataBits) val dataBeats = params(TLDataBeats) - val co = params(TLCoherence) val w = params(HTIFWidth) val nSCR = params(HTIFNSCR) val offsetBits = params(HTIFOffsetBits) @@ -197,12 +196,12 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { val init_addr = addr.toUInt >> UInt(offsetBits-3) io.mem.acquire.valid := state === state_mem_rreq || state === state_mem_wreq io.mem.acquire.bits.payload := Mux(cmd === cmd_writemem, - UncachedWriteBlock( + PutBlock( addr_block = init_addr, addr_beat = cnt, client_xact_id = UInt(0), data = mem_req_data), - UncachedReadBlock(addr_block = init_addr)) + GetBlock(addr_block = init_addr)) io.mem.acquire.bits.payload.data := mem_req_data io.mem.finish.valid := (state === state_mem_finish) && mem_needs_ack io.mem.finish.bits.payload.manager_xact_id := mem_gxid @@ -258,7 +257,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { for (i <- 0 until scr_rdata.size) scr_rdata(i) := io.scr.rdata(i) scr_rdata(0) := UInt(nCores) - scr_rdata(1) := UInt((BigInt(dataBits*dataBeats/8) << params(TLAddrBits)) >> 20) + scr_rdata(1) := UInt((BigInt(dataBits*dataBeats/8) << params(TLBlockAddrBits)) >> 20) io.scr.wen := Bool(false) io.scr.wdata := pcr_wdata diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index e58dd08d..f10b3257 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -202,13 +202,15 @@ class MemDesser(w: Int) extends Module // test rig side io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UInt(w)) } +//Adapter between a TileLinkIO and a UncachedTileLinkIO, merges voluntary + + //Adapter betweewn an UncachedTileLinkIO and a mem controller MemIO -class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { +class MemIOTileLinkIOConverter(qDepth: Int) extends Module { val io = new Bundle { - val uncached = new UncachedTileLinkIO().flip + val tl = new TileLinkIO().flip val mem = new MemIO } - val co = params(TLCoherence) val mifTagBits = params(MIFTagBits) val mifDataBits = params(MIFDataBits) val mifDataBeats = params(MIFDataBeats) @@ -216,121 +218,241 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { val tlDataBeats = params(TLDataBeats) val dataBits = tlDataBits*tlDataBeats require(tlDataBits*tlDataBeats == mifDataBits*mifDataBeats) - //require(params(TLClientXactIdBits) <= params(MIFTagBits)) + require(params(TLClientXactIdBits) <= params(MIFTagBits)) + + io.tl.acquire.ready := Bool(false) + io.tl.probe.valid := Bool(false) + io.tl.release.ready := Bool(false) + io.tl.finish.ready := Bool(true) + io.mem.resp.ready := Bool(false) + + val gnt_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), 2)) + io.tl.grant <> gnt_arb.io.out + + val acq_has_data = io.tl.acquire.bits.payload.hasData() + val rel_has_data = io.tl.release.bits.payload.hasData() // Decompose outgoing TL Acquires into MemIO cmd and data - val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth)) - val mem_data_q = Module(new Queue(new MemData, qDepth)) - - io.uncached.acquire.ready := Bool(false) - io.uncached.grant.valid := Bool(false) - io.mem.resp.ready := Bool(false) - mem_cmd_q.io.enq.valid := Bool(false) - mem_data_q.io.enq.valid := Bool(false) - - //TODO: Assert that only WriteUncachedBlock and ReadUncachedBlock are - //acceptable Acquire types - val acq_has_data = io.uncached.acquire.bits.payload.hasData() - val (tl_cnt_out, tl_wrap_out) = Counter(io.uncached.acquire.fire() && acq_has_data, tlDataBeats) - val (mif_cnt_out, mif_wrap_out) = Counter(mem_data_q.io.enq.fire(), mifDataBeats) val active_out = Reg(init=Bool(false)) val cmd_sent_out = Reg(init=Bool(false)) - val tl_done_out = Reg(init=Bool(false)) - val mif_done_out = Reg(init=Bool(false)) val tag_out = Reg(Bits()) val addr_out = Reg(Bits()) val has_data = Reg(init=Bool(false)) - val tl_buf_out = Vec.fill(tlDataBeats){ Reg(io.uncached.acquire.bits.payload.data.clone) } - val mif_buf_out = Vec.fill(mifDataBeats){ new MemData } - mif_buf_out := mif_buf_out.fromBits(tl_buf_out.toBits) - val mif_prog_out = (mif_cnt_out+UInt(1, width = log2Up(mifDataBeats+1)))*UInt(mifDataBits) - val tl_prog_out = tl_cnt_out*UInt(tlDataBits) + val data_from_rel = Reg(init=Bool(false)) + val (tl_cnt_out, tl_wrap_out) = + Counter((io.tl.acquire.fire() && acq_has_data) || + (io.tl.release.fire() && rel_has_data), tlDataBeats) + val tl_done_out = Reg(init=Bool(false)) + val make_grant_ack = Reg(init=Bool(false)) + val grant_for_rel = Grant( + is_builtin_type = Bool(true), + g_type = Grant.voluntaryAckType, + client_xact_id = tag_out, + manager_xact_id = UInt(0)) + val grant_for_acq_write = ManagerMetadata.onReset.makeGrant( + acq = Acquire( + is_builtin_type = tag_out(0), + a_type = tag_out >> UInt(1), + client_xact_id = tag_out >> UInt(io.tl.tlAcquireTypeBits+1), + addr_block = UInt(0)), //DNC + manager_xact_id = UInt(0)) + gnt_arb.io.in(1).valid := Bool(false) + gnt_arb.io.in(1).bits.payload := Mux(data_from_rel, grant_for_rel, grant_for_acq_write) - when(!active_out){ - io.uncached.acquire.ready := Bool(true) - when(io.uncached.acquire.valid) { - active_out := Bool(true) - cmd_sent_out := Bool(false) - tag_out := io.uncached.acquire.bits.payload.client_xact_id - addr_out := io.uncached.acquire.bits.payload.addr_block - has_data := acq_has_data - tl_done_out := tl_wrap_out - mif_done_out := Bool(false) - tl_buf_out(tl_cnt_out) := io.uncached.acquire.bits.payload.data + if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) { + val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth)) + val mem_data_q = Module(new Queue(new MemData, qDepth)) + mem_cmd_q.io.enq.valid := Bool(false) + mem_data_q.io.enq.valid := Bool(false) + val (mif_cnt_out, mif_wrap_out) = Counter(mem_data_q.io.enq.fire(), mifDataBeats) + val mif_done_out = Reg(init=Bool(false)) + val tl_buf_out = Vec.fill(tlDataBeats){ Reg(io.tl.acquire.bits.payload.data.clone) } + val mif_buf_out = Vec.fill(mifDataBeats){ new MemData } + mif_buf_out := mif_buf_out.fromBits(tl_buf_out.toBits) + val mif_prog_out = (mif_cnt_out+UInt(1, width = log2Up(mifDataBeats+1)))*UInt(mifDataBits) + val tl_prog_out = tl_cnt_out*UInt(tlDataBits) + + when(!active_out){ + io.tl.release.ready := Bool(true) + io.tl.acquire.ready := !io.tl.release.valid + when(io.tl.release.valid) { + active_out := Bool(true) + cmd_sent_out := Bool(false) + tag_out := io.tl.release.bits.payload.client_xact_id + addr_out := io.tl.release.bits.payload.addr_block + has_data := rel_has_data + data_from_rel := Bool(true) + make_grant_ack := Bool(true) + tl_done_out := tl_wrap_out + tl_buf_out(tl_cnt_out) := io.tl.release.bits.payload.data + } .elsewhen(io.tl.acquire.valid) { + active_out := Bool(true) + cmd_sent_out := Bool(false) + tag_out := Cat(io.tl.acquire.bits.payload.client_xact_id, + io.tl.acquire.bits.payload.a_type, + io.tl.acquire.bits.payload.is_builtin_type) + addr_out := io.tl.acquire.bits.payload.addr_block + has_data := acq_has_data + data_from_rel := Bool(false) + make_grant_ack := acq_has_data + tl_done_out := tl_wrap_out + tl_buf_out(tl_cnt_out) := io.tl.acquire.bits.payload.data + } } - } - when(active_out) { - when(!cmd_sent_out) { - mem_cmd_q.io.enq.valid := Bool(true) - } - when(has_data) { - when(!tl_done_out) { - io.uncached.acquire.ready := Bool(true) - when(io.uncached.acquire.valid) { - tl_buf_out(tl_cnt_out) := io.uncached.acquire.bits.payload.data + when(active_out) { + mem_cmd_q.io.enq.valid := !cmd_sent_out + cmd_sent_out := cmd_sent_out || mem_cmd_q.io.enq.fire() + when(has_data) { + when(!tl_done_out) { + io.tl.acquire.ready := Bool(true) + when(io.tl.acquire.valid) { + tl_buf_out(tl_cnt_out) := Mux(data_from_rel, + io.tl.release.bits.payload.data, + io.tl.acquire.bits.payload.data) + } + } + when(!mif_done_out) { + mem_data_q.io.enq.valid := tl_done_out || mif_prog_out <= tl_prog_out } } - when(!mif_done_out) { - mem_data_q.io.enq.valid := tl_done_out || mif_prog_out <= tl_prog_out + when(tl_wrap_out) { tl_done_out := Bool(true) } + when(mif_wrap_out) { mif_done_out := Bool(true) } + when(tl_done_out && make_grant_ack) { + gnt_arb.io.in(1).valid := Bool(true) + when(gnt_arb.io.in(1).ready) { make_grant_ack := Bool(false) } + } + when(cmd_sent_out && (!has_data || mif_done_out) && !make_grant_ack) { + active_out := Bool(false) } } - when(mem_cmd_q.io.enq.fire()) { - cmd_sent_out := Bool(true) + + mem_cmd_q.io.enq.bits.rw := has_data + mem_cmd_q.io.enq.bits.tag := tag_out + mem_cmd_q.io.enq.bits.addr := addr_out + mem_data_q.io.enq.bits.data := mif_buf_out(mif_cnt_out).data + io.mem.req_cmd <> mem_cmd_q.io.deq + io.mem.req_data <> mem_data_q.io.deq + } else { // Don't make the data buffers and try to flow cmd and data + io.mem.req_cmd.valid := Bool(false) + io.mem.req_data.valid := Bool(false) + io.mem.req_cmd.bits.rw := has_data + io.mem.req_cmd.bits.tag := tag_out + io.mem.req_cmd.bits.addr := addr_out + io.mem.req_data.bits.data := Mux(data_from_rel, + io.tl.release.bits.payload.data, + io.tl.acquire.bits.payload.data) + when(!active_out){ + io.tl.release.ready := io.mem.req_data.ready + io.tl.acquire.ready := io.mem.req_data.ready && !io.tl.release.valid + io.mem.req_data.valid := (io.tl.release.valid && rel_has_data) || + (io.tl.acquire.valid && acq_has_data) + when(io.mem.req_data.ready && (io.tl.release.valid || io.tl.acquire.valid)) { + active_out := !io.mem.req_cmd.ready || io.mem.req_data.valid + io.mem.req_cmd.valid := Bool(true) + cmd_sent_out := io.mem.req_cmd.ready + tag_out := io.mem.req_cmd.bits.tag + addr_out := io.mem.req_data.bits.data + has_data := io.mem.req_cmd.bits.rw + tl_done_out := tl_wrap_out + when(io.tl.release.valid) { + data_from_rel := Bool(true) + make_grant_ack := Bool(true) + io.mem.req_cmd.bits.rw := rel_has_data + io.mem.req_cmd.bits.tag := io.tl.release.bits.payload.client_xact_id + io.mem.req_cmd.bits.addr := io.tl.release.bits.payload.addr_block + io.mem.req_data.bits.data := io.tl.release.bits.payload.data + } .elsewhen(io.tl.acquire.valid) { + data_from_rel := Bool(false) + make_grant_ack := acq_has_data + io.mem.req_cmd.bits.rw := acq_has_data + io.mem.req_cmd.bits.tag := Cat(io.tl.acquire.bits.payload.client_xact_id, + io.tl.acquire.bits.payload.a_type, + io.tl.acquire.bits.payload.is_builtin_type) + io.mem.req_cmd.bits.addr := io.tl.acquire.bits.payload.addr_block + io.mem.req_data.bits.data := io.tl.acquire.bits.payload.data + } + } } - when(tl_wrap_out) { tl_done_out := Bool(true) } - when(mif_wrap_out) { mif_done_out := Bool(true) } - when(cmd_sent_out && (!has_data || mif_done_out)) { - active_out := Bool(false) + when(active_out) { + io.mem.req_cmd.valid := !cmd_sent_out + cmd_sent_out := cmd_sent_out || io.mem.req_cmd.fire() + when(has_data && !tl_done_out) { + when(data_from_rel) { + io.tl.release.ready := io.mem.req_data.ready + io.mem.req_data.valid := io.tl.release.valid + } .otherwise { + io.tl.acquire.ready := io.mem.req_data.ready + io.mem.req_data.valid := io.tl.acquire.valid + } + } + when(tl_wrap_out) { tl_done_out := Bool(true) } + when(tl_done_out && make_grant_ack) { + gnt_arb.io.in(1).valid := Bool(true) + when(gnt_arb.io.in(1).ready) { make_grant_ack := Bool(false) } + } + when(cmd_sent_out && (!has_data || tl_done_out) && !make_grant_ack) { + active_out := Bool(false) + } } } - mem_cmd_q.io.enq.bits.rw := has_data - mem_cmd_q.io.enq.bits.tag := tag_out - mem_cmd_q.io.enq.bits.addr := addr_out - mem_data_q.io.enq.bits.data := mif_buf_out(mif_cnt_out).data - io.mem.req_cmd <> mem_cmd_q.io.deq - io.mem.req_data <> mem_data_q.io.deq - // Aggregate incoming MemIO responses into TL Grants - val (mif_cnt_in, mif_wrap_in) = Counter(io.mem.resp.fire(), mifDataBeats) // TODO: Assumes all resps have data - val (tl_cnt_in, tl_wrap_in) = Counter(io.uncached.grant.fire(), tlDataBeats) val active_in = Reg(init=Bool(false)) - val mif_done_in = Reg(init=Bool(false)) + val (tl_cnt_in, tl_wrap_in) = Counter(io.tl.grant.fire() && io.tl.grant.bits.payload.hasMultibeatData(), tlDataBeats) val tag_in = Reg(UInt(width = mifTagBits)) - val mif_buf_in = Vec.fill(mifDataBeats){ Reg(new MemData) } - val tl_buf_in = Vec.fill(tlDataBeats){ io.uncached.acquire.bits.payload.data.clone } - tl_buf_in := tl_buf_in.fromBits(mif_buf_in.toBits) - val tl_prog_in = (tl_cnt_in+UInt(1, width = log2Up(tlDataBeats+1)))*UInt(tlDataBits) - val mif_prog_in = mif_cnt_in*UInt(mifDataBits) - when(!active_in) { - io.mem.resp.ready := Bool(true) - when(io.mem.resp.valid) { - active_in := Bool(true) - mif_done_in := mif_wrap_in - tag_in := io.mem.resp.bits.tag - mif_buf_in(tl_cnt_in).data := io.mem.resp.bits.data - } - } - - when(active_in) { - io.uncached.grant.valid := mif_done_in || tl_prog_in <= mif_prog_in - when(!mif_done_in) { + if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) { + val (mif_cnt_in, mif_wrap_in) = Counter(io.mem.resp.fire(), mifDataBeats) // TODO: Assumes all resps have data + val mif_done_in = Reg(init=Bool(false)) + val mif_buf_in = Vec.fill(mifDataBeats){ Reg(new MemData) } + val tl_buf_in = Vec.fill(tlDataBeats){ io.tl.acquire.bits.payload.data.clone } + tl_buf_in := tl_buf_in.fromBits(mif_buf_in.toBits) + val tl_prog_in = (tl_cnt_in+UInt(1, width = log2Up(tlDataBeats+1)))*UInt(tlDataBits) + val mif_prog_in = mif_cnt_in*UInt(mifDataBits) + gnt_arb.io.in(0).bits.payload := ManagerMetadata.onReset.makeGrant( + acq = Acquire( + is_builtin_type = tag_in(0), + a_type = tag_in >> UInt(1), + client_xact_id = tag_in >> UInt(io.tl.tlAcquireTypeBits+1), + addr_block = UInt(0)), //DNC + manager_xact_id = UInt(0), + addr_beat = tl_cnt_in, + data = tl_buf_in(tl_cnt_in)) + + when(!active_in) { io.mem.resp.ready := Bool(true) when(io.mem.resp.valid) { - mif_buf_in(mif_cnt_in).data := io.mem.resp.bits.data + active_in := Bool(true) + mif_done_in := mif_wrap_in + tag_in := io.mem.resp.bits.tag + mif_buf_in(tl_cnt_in).data := io.mem.resp.bits.data } } - when(mif_wrap_in) { mif_done_in := Bool(true) } - when(tl_wrap_in) { active_in := Bool(false) } + when(active_in) { + gnt_arb.io.in(0).valid := mif_done_in || tl_prog_in <= mif_prog_in + when(!mif_done_in) { + io.mem.resp.ready := Bool(true) + when(io.mem.resp.valid) { + mif_buf_in(mif_cnt_in).data := io.mem.resp.bits.data + } + } + when(mif_wrap_in) { mif_done_in := Bool(true) } + when(tl_wrap_in) { active_in := Bool(false) } + } + } else { // Don't generate all the uneeded data buffers and flow resp + gnt_arb.io.in(0).valid := io.mem.resp.valid + io.mem.resp.ready := gnt_arb.io.in(0).ready + gnt_arb.io.in(0).bits.payload := + ManagerMetadata.onReset.makeGrant( + acq = Acquire( + is_builtin_type = io.mem.resp.bits.tag(0), + a_type = io.mem.resp.bits.tag >> UInt(1), + client_xact_id = io.mem.resp.bits.tag >> UInt(io.tl.tlAcquireTypeBits+1), + addr_block = UInt(0)), //DNC + manager_xact_id = UInt(0), + addr_beat = tl_cnt_in, + data = io.mem.resp.bits.data) } - - io.uncached.grant.bits.payload := Grant(builtin_type = Bool(true), - g_type = Grant.uncachedReadBlock, - client_xact_id = tag_in, - manager_xact_id = UInt(0), - addr_beat = tl_cnt_in, - data = tl_buf_in(tl_cnt_in)) } class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module @@ -428,15 +550,15 @@ class MemPipeIOMemIOConverter(numRequests: Int, refillCycles: Int) extends Modul dec := io.mem.req_cmd.fire() && !io.mem.req_cmd.bits.rw } -class MemPipeIOUncachedTileLinkIOConverter(outstanding: Int, refillCycles: Int) extends Module { +class MemPipeIOTileLinkIOConverter(outstanding: Int, refillCycles: Int) extends Module { val io = new Bundle { - val uncached = new UncachedTileLinkIO().flip + val tl = new TileLinkIO().flip val mem = new MemPipeIO } - val a = Module(new MemIOUncachedTileLinkIOConverter(2)) + val a = Module(new MemIOTileLinkIOConverter(1)) val b = Module(new MemPipeIOMemIOConverter(outstanding, refillCycles)) - a.io.uncached <> io.uncached + a.io.tl <> io.tl b.io.cpu.req_cmd <> Queue(a.io.mem.req_cmd, 2, pipe=true) b.io.cpu.req_data <> Queue(a.io.mem.req_data, refillCycles, pipe=true) a.io.mem.resp <> b.io.cpu.resp diff --git a/uncore/src/main/scala/metadata.scala b/uncore/src/main/scala/metadata.scala new file mode 100644 index 00000000..8f5d7e90 --- /dev/null +++ b/uncore/src/main/scala/metadata.scala @@ -0,0 +1,188 @@ +// See LICENSE for license details. + +package uncore +import Chisel._ + +// Classes to represent coherence information in clients and managers +abstract class CoherenceMetadata extends Bundle { + val co = params(TLCoherencePolicy) + val id = params(TLId) +} + +/* The ClientMetadata stores the client-side coherence information, + such as permissions on the data and whether the data is dirty. + Its API can be used to make TileLink messages in response to + memory operations or TileLink Probes. +*/ +class ClientMetadata extends CoherenceMetadata { + val state = UInt(width = co.clientStateWidth) + + def ===(rhs: ClientMetadata): Bool = this.state === rhs.state + + def isValid(dummy: Int = 0): Bool = co.isValid(this) + def isHit(cmd: UInt): Bool = co.isHit(cmd, this) + def isMiss(cmd: UInt): Bool = !co.isHit(cmd, this) + def requiresAcquireOnSecondaryMiss(first_cmd: UInt, second_cmd: UInt): Bool = + co.requiresAcquireOnSecondaryMiss(first_cmd, second_cmd, this) + def requiresReleaseOnCacheControl(cmd: UInt): Bool = + co.requiresReleaseOnCacheControl(cmd: UInt, this) + def requiresVoluntaryWriteback(dummy: Int = 0): Bool = + co.requiresReleaseOnCacheControl(M_FLUSH, this) + + def makeAcquire( + client_xact_id: UInt, + addr_block: UInt, + op_code: UInt): Acquire = { + Bundle(Acquire( + is_builtin_type = Bool(false), + a_type = co.getAcquireType(op_code, this), + client_xact_id = client_xact_id, + addr_block = addr_block, + union = Cat(op_code, Bool(true))), + { case TLId => id }) + } + + def makeVoluntaryWriteback( + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Release = { + Bundle(Release( + voluntary = Bool(true), + r_type = co.getReleaseType(M_FLUSH, this), + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = addr_beat, + data = data), { case TLId => id }) + } + + def makeRelease( + prb: Probe, + client_xact_id: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Release = { + Bundle(Release( + voluntary = Bool(false), + r_type = co.getReleaseType(prb, this), + client_xact_id = client_xact_id, + addr_block = prb.addr_block, + addr_beat = addr_beat, + data = data), { case TLId => id }) + } + + def onGrant(incoming: Grant, pending: UInt): ClientMetadata = + Bundle(co.clientMetadataOnGrant(incoming, pending, this), { case TLId => id }) + def onProbe(incoming: Probe): ClientMetadata = + Bundle(co.clientMetadataOnProbe(incoming, this), { case TLId => id }) + def onHit(cmd: UInt): ClientMetadata = + Bundle(co.clientMetadataOnHit(cmd, this), { case TLId => id }) + def onCacheControl(cmd: UInt): ClientMetadata = + Bundle(co.clientMetadataOnCacheControl(cmd, this), { case TLId => id }) +} + +object ClientMetadata { + def apply(state: UInt) = { + val meta = new ClientMetadata + meta.state := state + meta + } + def onReset = new ClientMetadata().co.clientMetadataOnReset +} + +/* The ManagerMetadata stores manager-side information about the status + of a cache block, including whether it has any known sharers. Its + API can be used to create Probe and Grant TileLink messages. +*/ +class ManagerMetadata extends CoherenceMetadata { + // val state = UInt(width = co.masterStateWidth) TODO: Fix 0-width wires in Chisel + val sharers = UInt(width = co.dir.width) + + def ===(rhs: ManagerMetadata): Bool = //this.state === rhs.state && TODO: Fix 0-width wires in Chisel + this.sharers === rhs.sharers + def full(dummy: Int = 0) = co.dir.full(this.sharers) + + def requiresProbes(acq: Acquire): Bool = co.requiresProbes(acq, this) + def requiresProbes(cmd: UInt): Bool = co.requiresProbes(cmd, this) + def requiresProbesOnVoluntaryWriteback(dummy: Int = 0): Bool = + co.requiresProbes(M_FLUSH, this) + + def makeProbe(cmd: UInt, addr_block: UInt): Probe = + Bundle(Probe(co.getProbeType(cmd, this), addr_block), { case TLId => id }) + + def makeProbe(acq: Acquire): Probe = + Bundle(Probe(co.getProbeType(acq, this), acq.addr_block), { case TLId => id }) + + def makeProbeForVoluntaryWriteback(addr_block: UInt): Probe = + makeProbe(M_FLUSH, addr_block) + + def makeGrant(rel: Release, manager_xact_id: UInt): Grant = { + Bundle(Grant( + is_builtin_type = Bool(true), + g_type = Grant.voluntaryAckType, + client_xact_id = rel.client_xact_id, + manager_xact_id = manager_xact_id), { case TLId => id }) + } + + def makeGrant( + acq: Acquire, + manager_xact_id: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Grant = { + Bundle(Grant( + is_builtin_type = acq.isBuiltInType(), + g_type = Mux(acq.isBuiltInType(), + acq.getBuiltInGrantType(), + co.getGrantType(acq, this)), + client_xact_id = acq.client_xact_id, + manager_xact_id = manager_xact_id, + addr_beat = addr_beat, + data = data), { case TLId => id }) + } + + def onRelease(incoming: Release, src: UInt): ManagerMetadata = + Bundle(co.managerMetadataOnRelease(incoming, src, this), { case TLId => id }) + + def onGrant(outgoing: Grant, dst: UInt): ManagerMetadata = + Bundle(co.managerMetadataOnGrant(outgoing, dst, this), { case TLId => id }) +} + +object ManagerMetadata { + def apply(sharers: UInt, state: UInt = UInt(width = 0)) = { + val meta = new ManagerMetadata + //meta.state := state TODO: Fix 0-width wires in Chisel + meta.sharers := sharers + meta + } + def apply() = { + val meta = new ManagerMetadata + //meta.state := UInt(width = 0) TODO: Fix 0-width wires in Chisel + meta.sharers := meta.co.dir.flush + meta + } + def onReset = new ManagerMetadata().co.managerMetadataOnReset +} + +/* HierarchicalMetadata is used in a cache in a multi-level memory hierarchy + that is a Manager with respect to some inner caches and a Client with + respect to some outer cache. +*/ +class HierarchicalMetadata extends CoherenceMetadata { + val inner: ManagerMetadata = Bundle(new ManagerMetadata, {case TLId => params(InnerTLId)}) + val outer: ClientMetadata = Bundle(new ClientMetadata, {case TLId => params(OuterTLId)}) + def ===(rhs: HierarchicalMetadata): Bool = + this.inner === rhs.inner && this.outer === rhs.outer + def !=(rhs: HierarchicalMetadata): Bool = !(this === rhs) +} + +object HierarchicalMetadata { + def apply(inner: ManagerMetadata, outer: ClientMetadata): HierarchicalMetadata = { + val m = new HierarchicalMetadata + m.inner := inner + m.outer := outer + m + } + def onReset: HierarchicalMetadata = apply(ManagerMetadata.onReset, ClientMetadata.onReset) +} + +case object InnerTLId extends Field[String] +case object OuterTLId extends Field[String] diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index afbf4dc0..c8996ddf 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -8,30 +8,35 @@ import scala.math.max // external requirements or design space exploration // case object TLId extends Field[String] // Unique name per network -case object TLCoherence extends Field[CoherencePolicy] -case object TLAddrBits extends Field[Int] +case object TLCoherencePolicy extends Field[CoherencePolicy] +case object TLBlockAddrBits extends Field[Int] case object TLManagerXactIdBits extends Field[Int] case object TLClientXactIdBits extends Field[Int] case object TLDataBits extends Field[Int] case object TLDataBeats extends Field[Int] +case object TLNetworkIsOrderedP2P extends Field[Boolean] abstract trait TileLinkParameters extends UsesParameters { - val tlBlockAddrBits = params(TLAddrBits) + val tlBlockAddrBits = params(TLBlockAddrBits) val tlClientXactIdBits = params(TLClientXactIdBits) val tlManagerXactIdBits = params(TLManagerXactIdBits) val tlDataBits = params(TLDataBits) val tlDataBeats = params(TLDataBeats) + val tlCoh = params(TLCoherencePolicy) val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits/8 val tlBeatAddrBits = log2Up(tlDataBeats) val tlByteAddrBits = log2Up(tlWriteMaskBits) - val tlAtomicOpcodeBits = M_SZ - val tlUncachedOperandSizeBits = MT_SZ - val tlSubblockUnionBits = max(tlWriteMaskBits, + val tlMemoryOpcodeBits = M_SZ + val tlMemoryOperandSizeBits = MT_SZ + val tlAcquireTypeBits = max(log2Up(Acquire.nBuiltInTypes), + tlCoh.acquireTypeWidth) + val tlAcquireUnionBits = max(tlWriteMaskBits, (tlByteAddrBits + - tlUncachedOperandSizeBits + - tlAtomicOpcodeBits)) + 1 - val co = params(TLCoherence) - val networkPreservesPointToPointOrdering = false //TODO: check physical network type + tlMemoryOperandSizeBits + + tlMemoryOpcodeBits)) + 1 + val tlGrantTypeBits = max(log2Up(Grant.nBuiltInTypes), + tlCoh.grantTypeWidth) + 1 + val tlNetworkPreservesPointToPointOrdering = params(TLNetworkIsOrderedP2P) } abstract class TLBundle extends Bundle with TileLinkParameters @@ -50,8 +55,8 @@ trait ClientToClientChannel extends TileLinkChannel // Unused for now trait HasCacheBlockAddress extends TLBundle { val addr_block = UInt(width = tlBlockAddrBits) - def conflicts[T <: HasCacheBlockAddress](that: T) = this.addr_block === that.addr_block - def conflicts[T <: HasCacheBlockAddress](addr: UInt) = this.addr_block === addr + def conflicts(that: HasCacheBlockAddress) = this.addr_block === that.addr_block + def conflicts(addr: UInt) = this.addr_block === addr } trait HasTileLinkBeatId extends TLBundle { @@ -66,8 +71,9 @@ trait HasManagerTransactionId extends TLBundle { val manager_xact_id = Bits(width = tlManagerXactIdBits) } -abstract trait HasTileLinkData extends HasTileLinkBeatId { +trait HasTileLinkData extends HasTileLinkBeatId { val data = UInt(width = tlDataBits) + def hasData(dummy: Int = 0): Bool def hasMultibeatData(dummy: Int = 0): Bool } @@ -79,95 +85,81 @@ class Acquire extends ClientToManagerChannel with HasClientTransactionId with HasTileLinkData { // Actual bundle fields - val builtin_type = Bool() - val a_type = UInt(width = max(log2Up(Acquire.nBuiltinAcquireTypes), co.acquireTypeWidth)) - val subblock = Bits(width = tlSubblockUnionBits) + val is_builtin_type = Bool() + val a_type = UInt(width = tlAcquireTypeBits) + val union = Bits(width = tlAcquireUnionBits) // Utility funcs for accessing subblock union - val opSizeOff = tlByteAddrBits + 1 - val opCodeOff = tlUncachedOperandSizeBits + opSizeOff - val opMSB = tlAtomicOpcodeBits + opCodeOff - def allocate(dummy: Int = 0) = subblock(0) - def addr_byte(dummy: Int = 0) = subblock(opSizeOff-1, 1) - def op_size(dummy: Int = 0) = subblock(opCodeOff-1, opSizeOff) - def op_code(dummy: Int = 0) = subblock(opMSB-1, opCodeOff) - def write_mask(dummy: Int = 0) = subblock(tlWriteMaskBits, 1) - def addr(dummy: Int = 0) = Cat(addr_block, addr_beat, this.addr_byte(0)) + val opCodeOff = 1 + val opSizeOff = tlMemoryOpcodeBits + opCodeOff + val addrByteOff = tlMemoryOperandSizeBits + opSizeOff + val addrByteMSB = tlByteAddrBits + addrByteOff + def allocate(dummy: Int = 0) = union(0) + def op_code(dummy: Int = 0) = Mux(hasData(), M_XWR, union(opSizeOff-1, opCodeOff)) + def op_size(dummy: Int = 0) = union(addrByteOff-1, opSizeOff) + def addr_byte(dummy: Int = 0) = union(addrByteMSB-1, addrByteOff) + def write_mask(dummy: Int = 0) = union(tlWriteMaskBits, 1) + def addr(dummy: Int = 0) = Cat(this.addr_block, this.addr_beat, this.addr_byte()) // Other helper funcs def is(t: UInt) = a_type === t - def hasData(dummy: Int = 0): Bool = builtin_type && Acquire.typesWithData.contains(a_type) + def isBuiltInType(dummy: Int = 0): Bool = is_builtin_type - def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && builtin_type && + def isSubBlockType(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesOnSubBlocks.contains(a_type) + + // Assumes no custom types have data + def hasData(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesWithData.contains(a_type) + + def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && isBuiltInType() && Acquire.typesWithMultibeatData.contains(a_type) //TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache: - def requiresSelfProbe(dummy: Int = 0) = builtin_type && Acquire.requiresSelfProbe(a_type) + def requiresSelfProbe(dummy: Int = 0) = isBuiltInType() && a_type === Acquire.getBlockType - def makeProbe(meta: ManagerMetadata = co.managerMetadataOnFlush): Probe = - Probe(co.getProbeType(this, meta), this.addr_block) - - def makeGrant( - manager_xact_id: UInt, - meta: ManagerMetadata = co.managerMetadataOnFlush, - addr_beat: UInt = UInt(0), - data: UInt = UInt(0)): Grant = { - Grant( - builtin_type = this.builtin_type, - g_type = co.getGrantType(this, meta), - client_xact_id = this.client_xact_id, - manager_xact_id = manager_xact_id, - addr_beat = addr_beat, - data = data - ) + def getBuiltInGrantType(dummy: Int = 0): UInt = { + MuxLookup(this.a_type, Grant.ackType, Array( + Acquire.getType -> Grant.dataBeatType, + Acquire.getBlockType -> Grant.dataBlockType, + Acquire.putType -> Grant.ackType, + Acquire.putBlockType -> Grant.ackType, + Acquire.putAtomicType -> Grant.dataBeatType)) } } object Acquire { - val nBuiltinAcquireTypes = 5 + val nBuiltInTypes = 5 //TODO: Use Enum - def uncachedRead = UInt(0) - def uncachedReadBlock = UInt(1) - def uncachedWrite = UInt(2) - def uncachedWriteBlock = UInt(3) - def uncachedAtomic = UInt(4) - def typesWithData = Vec(uncachedWrite, uncachedWriteBlock, uncachedAtomic) - def typesWithMultibeatData = Vec(uncachedWriteBlock) - def requiresOuterRead(a_type: UInt) = a_type != uncachedWriteBlock - def requiresOuterWrite(a_type: UInt) = typesWithData.contains(a_type) - //TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache: - def requiresSelfProbe(a_type: UInt) = a_type === uncachedReadBlock + def getType = UInt("b000") + def getBlockType = UInt("b001") + def putType = UInt("b010") + def putBlockType = UInt("b011") + def putAtomicType = UInt("b100") + def typesWithData = Vec(putType, putBlockType, putAtomicType) + def typesWithMultibeatData = Vec(putBlockType) + def typesOnSubBlocks = Vec(putType, getType, putAtomicType) def fullWriteMask = SInt(-1, width = new Acquire().tlWriteMaskBits).toUInt // Most generic constructor def apply( - builtin_type: Bool, + is_builtin_type: Bool, a_type: Bits, client_xact_id: UInt, addr_block: UInt, addr_beat: UInt = UInt(0), data: UInt = UInt(0), - subblock: UInt = UInt(0)): Acquire = { + union: UInt = UInt(0)): Acquire = { val acq = new Acquire - acq.builtin_type := builtin_type + acq.is_builtin_type := is_builtin_type acq.a_type := a_type acq.client_xact_id := client_xact_id acq.addr_block := addr_block acq.addr_beat := addr_beat acq.data := data - acq.subblock := subblock + acq.union := union acq } - // For cached types - def apply(a_type: Bits, client_xact_id: UInt, addr_block: UInt): Acquire = { - apply( - builtin_type = Bool(false), - a_type = a_type, - client_xact_id = client_xact_id, - addr_block = addr_block) - } // Copy constructor def apply(a: Acquire): Acquire = { val acq = new Acquire @@ -177,58 +169,73 @@ object Acquire { } // Asks for a single TileLink beat of data -object UncachedRead { +object Get { def apply( client_xact_id: UInt, addr_block: UInt, addr_beat: UInt, alloc: Bool = Bool(true)): Acquire = { Acquire( - builtin_type = Bool(true), - a_type = Acquire.uncachedRead, + is_builtin_type = Bool(true), + a_type = Acquire.getType, client_xact_id = client_xact_id, addr_block = addr_block, addr_beat = addr_beat, - subblock = alloc) + union = Cat(M_XRD, alloc)) } } // Asks for an entire cache block of data -object UncachedReadBlock { +object GetBlock { def apply( client_xact_id: UInt = UInt(0), addr_block: UInt, alloc: Bool = Bool(true)): Acquire = { Acquire( - builtin_type = Bool(true), - a_type = Acquire.uncachedReadBlock, + is_builtin_type = Bool(true), + a_type = Acquire.getBlockType, client_xact_id = client_xact_id, addr_block = addr_block, - subblock = alloc.toUInt) + union = Cat(M_XRD, alloc)) } } -object UncachedWrite { +// Writes up to a single TileLink beat of data, using mask +object Put { def apply( client_xact_id: UInt, addr_block: UInt, addr_beat: UInt, data: UInt, - write_mask: UInt = Acquire.fullWriteMask, - alloc: Bool = Bool(true)): Acquire = { + write_mask: UInt = Acquire.fullWriteMask): Acquire = { Acquire( - builtin_type = Bool(true), - a_type = Acquire.uncachedWrite, + is_builtin_type = Bool(true), + a_type = Acquire.putType, addr_block = addr_block, addr_beat = addr_beat, client_xact_id = client_xact_id, data = data, - subblock = Cat(write_mask, alloc)) + union = Cat(write_mask, Bool(true))) } } -// For full block of data -object UncachedWriteBlock { +// Writes an entire cache block of data +object PutBlock { + def apply( + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, + data: UInt, + write_mask: UInt): Acquire = { + Acquire( + is_builtin_type = Bool(true), + a_type = Acquire.putBlockType, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = addr_beat, + data = data, + union = Cat(write_mask, (write_mask != Acquire.fullWriteMask))) + } def apply( client_xact_id: UInt, addr_block: UInt, @@ -236,17 +243,18 @@ object UncachedWriteBlock { data: UInt, alloc: Bool = Bool(true)): Acquire = { Acquire( - builtin_type = Bool(true), - a_type = Acquire.uncachedWriteBlock, + is_builtin_type = Bool(true), + a_type = Acquire.putBlockType, client_xact_id = client_xact_id, addr_block = addr_block, addr_beat = addr_beat, data = data, - subblock = Cat(Acquire.fullWriteMask, alloc)) + union = Cat(Acquire.fullWriteMask, alloc)) } } -object UncachedAtomic { +// Performs an atomic operation in the outer memory +object PutAtomic { def apply( client_xact_id: UInt, addr_block: UInt, @@ -256,48 +264,30 @@ object UncachedAtomic { operand_size: UInt, data: UInt): Acquire = { Acquire( - builtin_type = Bool(true), - a_type = Acquire.uncachedAtomic, + is_builtin_type = Bool(true), + a_type = Acquire.putAtomicType, client_xact_id = client_xact_id, addr_block = addr_block, addr_beat = addr_beat, data = data, - subblock = Cat(atomic_opcode, operand_size, addr_byte, Bool(true))) + union = Cat(addr_byte, operand_size, atomic_opcode, Bool(true))) } } class Probe extends ManagerToClientChannel with HasCacheBlockAddress { - val p_type = UInt(width = co.probeTypeWidth) + val p_type = UInt(width = tlCoh.probeTypeWidth) def is(t: UInt) = p_type === t - def makeRelease( - client_xact_id: UInt, - meta: ClientMetadata = co.clientMetadataOnFlush, - addr_beat: UInt = UInt(0), - data: UInt = UInt(0)): Release = { - Release( - voluntary = Bool(false), - r_type = co.getReleaseType(this, meta), - client_xact_id = client_xact_id, - addr_block = this.addr_block, - addr_beat = addr_beat, - data = data) - } } object Probe { - val co = new Probe().co def apply(p_type: UInt, addr_block: UInt) = { val prb = new Probe prb.p_type := p_type prb.addr_block := addr_block prb } - - def onVoluntaryWriteback(meta: ManagerMetadata, addr_block: UInt): Probe = { - apply(co.getProbeType(M_FLUSH, meta), addr_block) - } } @@ -305,31 +295,19 @@ class Release extends ClientToManagerChannel with HasCacheBlockAddress with HasClientTransactionId with HasTileLinkData { - val r_type = UInt(width = co.releaseTypeWidth) + val r_type = UInt(width = tlCoh.releaseTypeWidth) val voluntary = Bool() // Helper funcs def is(t: UInt) = r_type === t - def hasData(dummy: Int = 0) = co.releaseTypesWithData.contains(r_type) + def hasData(dummy: Int = 0) = tlCoh.releaseTypesWithData.contains(r_type) //TODO: Assumes all releases write back full cache blocks: - def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) && co.releaseTypesWithData.contains(r_type) + def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) && tlCoh.releaseTypesWithData.contains(r_type) def isVoluntary(dummy: Int = 0) = voluntary - def requiresAck(dummy: Int = 0) = !Bool(networkPreservesPointToPointOrdering) - - def makeGrant( - manager_xact_id: UInt, - meta: ManagerMetadata = co.managerMetadataOnFlush): Grant = { - Grant( - g_type = Grant.voluntaryAck, - builtin_type = Bool(true), // Grant.voluntaryAck is built-in type - client_xact_id = this.client_xact_id, - manager_xact_id = manager_xact_id - ) - } + def requiresAck(dummy: Int = 0) = !Bool(tlNetworkPreservesPointToPointOrdering) } object Release { - val co = new Release().co def apply( voluntary: Bool, r_type: UInt, @@ -346,68 +324,52 @@ object Release { rel.voluntary := voluntary rel } - - def makeVoluntaryWriteback( - meta: ClientMetadata, - client_xact_id: UInt, - addr_block: UInt, - addr_beat: UInt = UInt(0), - data: UInt = UInt(0)): Release = { - Release( - voluntary = Bool(true), - r_type = co.getReleaseType(M_FLUSH, meta), - client_xact_id = client_xact_id, - addr_block = addr_block, - addr_beat = addr_beat, - data = data) - } } class Grant extends ManagerToClientChannel with HasTileLinkData with HasClientTransactionId with HasManagerTransactionId { - val builtin_type = Bool() - val g_type = UInt(width = max(log2Up(Grant.nBuiltinGrantTypes), co.grantTypeWidth)) + val is_builtin_type = Bool() + val g_type = UInt(width = tlGrantTypeBits) // Helper funcs - def is(t: UInt) = g_type === t - def hasData(dummy: Int = 0): Bool = Mux(builtin_type, + def isBuiltInType(dummy: Int = 0): Bool = is_builtin_type + def is(t: UInt):Bool = g_type === t + def hasData(dummy: Int = 0): Bool = Mux(isBuiltInType(), Grant.typesWithData.contains(g_type), - co.grantTypesWithData.contains(g_type)) + tlCoh.grantTypesWithData.contains(g_type)) def hasMultibeatData(dummy: Int = 0): Bool = - Bool(tlDataBeats > 1) && Mux(builtin_type, + Bool(tlDataBeats > 1) && Mux(isBuiltInType(), Grant.typesWithMultibeatData.contains(g_type), - co.grantTypesWithData.contains(g_type)) - def isVoluntary(dummy: Int = 0): Bool = builtin_type && (g_type === Grant.voluntaryAck) - def requiresAck(dummy: Int = 0): Bool = !Bool(networkPreservesPointToPointOrdering) && !isVoluntary() + tlCoh.grantTypesWithData.contains(g_type)) + def isVoluntary(dummy: Int = 0): Bool = isBuiltInType() && (g_type === Grant.voluntaryAckType) + def requiresAck(dummy: Int = 0): Bool = !Bool(tlNetworkPreservesPointToPointOrdering) && !isVoluntary() def makeFinish(dummy: Int = 0): Finish = { - val f = new Finish + val f = Bundle(new Finish, { case TLManagerXactIdBits => tlManagerXactIdBits }) f.manager_xact_id := this.manager_xact_id f } } object Grant { - val nBuiltinGrantTypes = 5 - //TODO Use Enum - def voluntaryAck = UInt(0) - def uncachedRead = UInt(1) - def uncachedReadBlock = UInt(2) - def uncachedWrite = UInt(3) - def uncachedAtomic = UInt(4) - def typesWithData = Vec(uncachedRead, uncachedReadBlock, uncachedAtomic) - def typesWithMultibeatData= Vec(uncachedReadBlock) + val nBuiltInTypes = 4 + def voluntaryAckType = UInt("b00") + def ackType = UInt("b01") + def dataBeatType = UInt("b10") + def dataBlockType = UInt("b11") + def typesWithData = Vec(dataBlockType, dataBeatType) + def typesWithMultibeatData= Vec(dataBlockType) def apply( - builtin_type: Bool, + is_builtin_type: Bool, g_type: UInt, client_xact_id: UInt, manager_xact_id: UInt, addr_beat: UInt = UInt(0), data: UInt = UInt(0)): Grant = { val gnt = new Grant - gnt.builtin_type := builtin_type + gnt.is_builtin_type := is_builtin_type gnt.g_type := g_type gnt.client_xact_id := client_xact_id gnt.manager_xact_id := manager_xact_id @@ -415,23 +377,12 @@ object Grant { gnt.data := data gnt } - - def getGrantTypeForUncached(a: Acquire): UInt = { - MuxLookup(a.a_type, Grant.uncachedRead, Array( - Acquire.uncachedRead -> Grant.uncachedRead, - Acquire.uncachedReadBlock -> Grant.uncachedReadBlock, - Acquire.uncachedWrite -> Grant.uncachedWrite, - Acquire.uncachedWriteBlock -> Grant.uncachedWrite, - Acquire.uncachedAtomic -> Grant.uncachedAtomic - )) - } } class Finish extends ClientToManagerChannel with HasManagerTransactionId - // Complete IO definitions for two types of TileLink clients -class UncachedTileLinkIO extends Bundle { +class UncachedTileLinkIO extends TLBundle { val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) @@ -456,11 +407,17 @@ class TileLinkIOWrapper extends TLModule { io.out.release.valid := Bool(false) } object TileLinkIOWrapper { - def apply[T <: Data](utl: UncachedTileLinkIO) = { + def apply(utl: UncachedTileLinkIO, p: Parameters): TileLinkIO = { + val conv = Module(new TileLinkIOWrapper)(p) + conv.io.in <> utl + conv.io.out + } + def apply(utl: UncachedTileLinkIO): TileLinkIO = { val conv = Module(new TileLinkIOWrapper) conv.io.in <> utl conv.io.out } + def apply(tl: TileLinkIO) = tl } abstract trait HasArbiterTypes { diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 1230771c..3c334376 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -2,437 +2,147 @@ package uncore import Chisel._ +import scala.reflect.ClassTag case object NReleaseTransactors extends Field[Int] case object NProbeTransactors extends Field[Int] case object NAcquireTransactors extends Field[Int] case object NIncoherentClients extends Field[Int] case object NCoherentClients extends Field[Int] -case object L2StoreDataQueueDepth extends Field[Int] -case object L2CoherencePolicy extends Field[DirectoryRepresentation => CoherencePolicy] -case object L2DirectoryRepresentation extends Field[DirectoryRepresentation] +case object L2CoherencePolicy extends Field[CoherencePolicy] -abstract trait CoherenceAgentParameters extends UsesParameters - with TileLinkParameters { +trait CoherenceAgentParameters extends UsesParameters { val nReleaseTransactors = 1 val nAcquireTransactors = params(NAcquireTransactors) val nTransactors = nReleaseTransactors + nAcquireTransactors val nCoherentClients = params(NCoherentClients) val nIncoherentClients = params(NIncoherentClients) val nClients = nCoherentClients + nIncoherentClients - val sdqDepth = params(L2StoreDataQueueDepth)*tlDataBeats - val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(sdqDepth)) - val nDataQueueLocations = 3 //Stores, VoluntaryWBs, Releases + def outerTLParams = params.alterPartial({ case TLId => params(OuterTLId)}) + val outerDataBeats = outerTLParams(TLDataBeats) + val outerDataBits = outerTLParams(TLDataBits) + def innerTLParams = params.alterPartial({case TLId => params(InnerTLId)}) + val innerDataBeats = innerTLParams(TLDataBeats) + val innerDataBits = innerTLParams(TLDataBits) + val innerBeatAddrBits = log2Up(innerDataBeats) + val innerByteAddrBits = log2Up(innerDataBits/8) + require(outerDataBeats == innerDataBeats) //TODO: must fix all xact_data Vecs to remove this requirement } +abstract class CoherenceAgentBundle extends Bundle with CoherenceAgentParameters +abstract class CoherenceAgentModule extends Module with CoherenceAgentParameters -class DataQueueLocation extends Bundle with CoherenceAgentParameters { - val idx = UInt(width = dqIdxBits) - val loc = UInt(width = log2Ceil(nDataQueueLocations)) -} -object DataQueueLocation { - def apply(idx: UInt, loc: UInt) = { - val d = new DataQueueLocation - d.idx := idx - d.loc := loc - d +trait HasCoherenceAgentWiringHelpers { + def doOutputArbitration[T <: Data : ClassTag]( + out: DecoupledIO[T], + ins: Seq[DecoupledIO[T]]) { + val arb = Module(new RRArbiter(out.bits.clone, ins.size)) + out <> arb.io.out + arb.io.in zip ins map { case (a, in) => a <> in } + } + + def doOutputArbitration[T <: HasTileLinkData : ClassTag, S <: LogicalNetworkIO[T] : ClassTag]( + out: DecoupledIO[S], + ins: Seq[DecoupledIO[S]]) { + def lock(o: LogicalNetworkIO[T]) = o.payload.hasMultibeatData() + val arb = Module(new LockingRRArbiter( + out.bits.clone, + ins.size, + out.bits.payload.tlDataBeats, + lock _)) + out <> arb.io.out + arb.io.in zip ins map { case (a, in) => a <> in } + } + + def doInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { + val idx = in.bits.id + outs.map(_.bits := in.bits) + outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && idx === UInt(i) } + } + + def doInputRouting[T <: HasManagerTransactionId]( + in: DecoupledIO[LogicalNetworkIO[T]], + outs: Seq[DecoupledIO[LogicalNetworkIO[T]]]) { + val idx = in.bits.payload.manager_xact_id + outs.map(_.bits := in.bits) + outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && idx === UInt(i) } + in.ready := Vec(outs.map(_.ready)).read(idx) } } -abstract class CoherenceAgent(innerId: String, outerId: String) extends Module - with CoherenceAgentParameters { - val io = new Bundle { - val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip - val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) - val incoherent = Vec.fill(nClients){Bool()}.asInput - } +trait HasInnerTLIO extends CoherenceAgentBundle { + val inner = Bundle(new TileLinkIO)(innerTLParams).flip + val incoherent = Vec.fill(nClients){Bool()}.asInput + def iacq(dummy: Int = 0) = inner.acquire.bits.payload + def iprb(dummy: Int = 0) = inner.probe.bits.payload + def irel(dummy: Int = 0) = inner.release.bits.payload + def ignt(dummy: Int = 0) = inner.grant.bits.payload + def ifin(dummy: Int = 0) = inner.finish.bits.payload } -class L2BroadcastHub(bankId: Int, innerId: String, outerId: String) extends - CoherenceAgent(innerId, outerId) { - - val internalDataBits = new DataQueueLocation().getWidth - val inStoreQueue :: inVolWBQueue :: inClientReleaseQueue :: Nil = Enum(UInt(), nDataQueueLocations) - - // Create SHRs for outstanding transactions - val trackerList = (0 until nReleaseTransactors).map(id => - Module(new VoluntaryReleaseTracker(id, bankId, innerId, outerId), {case TLDataBits => internalDataBits})) ++ - (nReleaseTransactors until nTransactors).map(id => - Module(new AcquireTracker(id, bankId, innerId, outerId), {case TLDataBits => internalDataBits})) - - // Propagate incoherence flags - trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) - - // Queue to store impending UncachedWrite data - val acquire = io.inner.acquire - val sdq_val = Reg(init=Bits(0, sdqDepth)) - val sdq_alloc_id = PriorityEncoder(~sdq_val) - val sdq_rdy = !sdq_val.andR - val sdq_enq = acquire.fire() && acquire.bits.payload.hasData() - val sdq = Vec.fill(sdqDepth){ Reg(io.inner.acquire.bits.payload.data) } - when (sdq_enq) { sdq(sdq_alloc_id) := acquire.bits.payload.data } - - // Handle acquire transaction initiation - val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - val block_acquires = any_acquire_conflict - val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) - for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.inner - alloc_arb.io.in(i).valid := t.acquire.ready - t.acquire.bits := acquire.bits - t.acquire.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits - t.acquire.valid := alloc_arb.io.in(i).ready - } - acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && sdq_rdy && !block_acquires - alloc_arb.io.out.ready := acquire.valid && sdq_rdy && !block_acquires - - // Queue to store impending Voluntary Release data - val release = io.inner.release - val voluntary = release.bits.payload.isVoluntary() - val vwbdq_enq = release.fire() && voluntary && release.bits.payload.hasData() - val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, tlDataBeats) //TODO Zero width - val vwbdq = Vec.fill(tlDataBeats){ Reg(release.bits.payload.data) } //TODO Assumes nReleaseTransactors == 1 - when(vwbdq_enq) { vwbdq(rel_data_cnt) := release.bits.payload.data } - - // Handle releases, which might be voluntary and might have data - val release_idx = Vec(trackerList.map(_.io.has_release_match)).indexWhere{b: Bool => b} - for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.inner - t.release.bits := release.bits - t.release.bits.payload.data := (if (i < nReleaseTransactors) - DataQueueLocation(rel_data_cnt, inVolWBQueue) - else DataQueueLocation(UInt(0), inClientReleaseQueue)).toBits - t.release.valid := release.valid && (release_idx === UInt(i)) - } - release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) - - // Wire finished transaction acks - val ack = io.inner.finish - trackerList.map(_.io.inner.finish.valid := ack.valid) - trackerList.map(_.io.inner.finish.bits := ack.bits) - ack.ready := Bool(true) - - // Wire probe requests to clients - val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) - io.inner.probe <> probe_arb.io.out - probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.probe } - - // Wire grant reply to initiating client - def hasData(m: LogicalNetworkIO[Grant]) = m.payload.hasMultibeatData() - val grant_arb = Module(new LockingArbiter(new LogicalNetworkIO(new Grant), trackerList.size, tlDataBeats, Some(hasData _))) - io.inner.grant.bits.payload.data := io.outer.grant.bits.payload.data - io.inner.grant.bits.payload.addr_beat := io.outer.grant.bits.payload.addr_beat - io.inner.grant <> grant_arb.io.out - grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } - - // Create an arbiter for the one memory port - val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), - { case TLId => outerId; case TLDataBits => internalDataBits }) - outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } - val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.payload.data) - val is_in_sdq = outer_data_ptr.loc === inStoreQueue - val free_sdq = io.outer.acquire.fire() && - io.outer.acquire.bits.payload.hasData() && - outer_data_ptr.loc === inStoreQueue - io.outer.acquire.bits.payload.data := MuxLookup(outer_data_ptr.loc, release.bits.payload.data, Array( - inStoreQueue -> sdq(outer_data_ptr.idx), - inVolWBQueue -> vwbdq(outer_data_ptr.idx))) - io.outer <> outer_arb.io.out - - // Update SDQ valid bits - when (io.outer.acquire.valid || sdq_enq) { - sdq_val := sdq_val & ~(UIntToOH(outer_data_ptr.idx) & Fill(sdqDepth, free_sdq)) | - PriorityEncoderOH(~sdq_val(sdqDepth-1,0)) & Fill(sdqDepth, sdq_enq) - } +trait HasUncachedOuterTLIO extends CoherenceAgentBundle { + val outer = Bundle(new UncachedTileLinkIO)(outerTLParams) + def oacq(dummy: Int = 0) = outer.acquire.bits.payload + def ognt(dummy: Int = 0) = outer.grant.bits.payload + def ofin(dummy: Int = 0) = outer.finish.bits.payload } - -abstract class XactTracker(innerId: String, outerId: String) extends Module { - val (co, tlDataBeats) = (params(TLCoherence), params(TLDataBeats)) - val nClients = params(NCoherentClients) + params(NIncoherentClients) - val io = new Bundle { - val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip - val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) - val tile_incoherent = Bits(INPUT, nClients) - val has_acquire_conflict = Bool(OUTPUT) - val has_release_match = Bool(OUTPUT) - } - - val cacq = io.inner.acquire.bits - val crel = io.inner.release.bits - val cgnt = io.inner.grant.bits - val cfin = io.inner.finish.bits - val macq = io.outer.acquire.bits - val mgnt = io.outer.grant.bits - +trait HasCachedOuterTLIO extends CoherenceAgentBundle { + val outer = Bundle(new TileLinkIO)(outerTLParams) + def oacq(dummy: Int = 0) = outer.acquire.bits.payload + def oprb(dummy: Int = 0) = outer.probe.bits.payload + def orel(dummy: Int = 0) = outer.release.bits.payload + def ognt(dummy: Int = 0) = outer.grant.bits.payload + def ofin(dummy: Int = 0) = outer.finish.bits.payload } -class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { - val s_idle :: s_outer :: s_grant :: s_ack :: Nil = Enum(UInt(), 4) - val state = Reg(init=s_idle) +class ManagerTLIO extends HasInnerTLIO with HasUncachedOuterTLIO - val xact_src = Reg(io.inner.release.bits.header.src.clone) - val xact_r_type = Reg(io.inner.release.bits.payload.r_type) - val xact_addr_block = Reg(io.inner.release.bits.payload.addr_block.clone) - val xact_client_xact_id = Reg(io.inner.release.bits.payload.client_xact_id.clone) - val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) } - val xact = Release( - voluntary = Bool(true), - r_type = xact_r_type, - client_xact_id = xact_client_xact_id, - addr_block = xact_addr_block) - - val collect_inner_data = Reg(init=Bool(false)) - // TODO: assert that all releases have full blocks of data - val (inner_data_cnt, inner_data_done) = - Counter(io.inner.release.fire() && io.inner.release.bits.payload.hasMultibeatData(), tlDataBeats) - val (outer_data_cnt, outer_data_done) = - Counter(io.outer.acquire.fire() && io.outer.acquire.bits.payload.hasMultibeatData(), tlDataBeats) - - io.has_acquire_conflict := Bool(false) - io.has_release_match := crel.payload.isVoluntary() - - io.outer.grant.ready := Bool(false) - io.outer.acquire.valid := Bool(false) - io.inner.acquire.ready := Bool(false) - io.inner.probe.valid := Bool(false) - io.inner.release.ready := Bool(false) - io.inner.grant.valid := Bool(false) - io.inner.finish.ready := Bool(false) - - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := xact.makeGrant(UInt(trackerId)) - - io.outer.acquire.bits.payload := Bundle(UncachedWriteBlock( - client_xact_id = UInt(trackerId), - addr_block = xact_addr_block, - addr_beat = outer_data_cnt, - data = xact_data(outer_data_cnt)), - { case TLId => outerId }) - - when(collect_inner_data) { - io.inner.release.ready := Bool(true) - when(io.inner.release.valid) { - xact_data(inner_data_cnt) := crel.payload.data - } - when(inner_data_done) { collect_inner_data := Bool(false) } - } - - switch (state) { - is(s_idle) { - io.inner.release.ready := Bool(true) - when( io.inner.release.valid ) { - xact_src := crel.header.src - xact_r_type := crel.payload.r_type - xact_addr_block := crel.payload.addr_block - xact_client_xact_id := crel.payload.client_xact_id - xact_data(UInt(0)) := crel.payload.data - collect_inner_data := crel.payload.hasMultibeatData() - state := Mux(crel.payload.hasData(), s_outer, - Mux(crel.payload.requiresAck(), s_ack, s_idle)) - } - } - is(s_outer) { - io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) - else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) - when(outer_data_done) { - state := Mux(xact.requiresAck(), s_grant, s_idle) - } - } - is(s_grant) { - io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { - state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) - } - } - is(s_ack) { - // TODO: This state is unnecessary if no client will ever issue the - // pending Acquire that caused this writeback until it receives the - // Grant.voluntaryAck for this writeback - io.inner.finish.ready := Bool(true) - when(io.inner.finish.valid) { state := s_idle } - } - } +abstract class CoherenceAgent extends CoherenceAgentModule { + def innerTL: TileLinkIO + def outerTL: TileLinkIO + def incoherent: Vec[Bool] } -class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { - val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_mem_resp :: s_ack :: Nil = Enum(UInt(), 7) - val state = Reg(init=s_idle) +abstract class ManagerCoherenceAgent extends CoherenceAgent + with HasCoherenceAgentWiringHelpers { + val io = new ManagerTLIO + def innerTL = io.inner + def outerTL = TileLinkIOWrapper(io.outer, outerTLParams) + def incoherent = io.incoherent +} - val xact_src = Reg(io.inner.acquire.bits.header.src.clone) - val xact_builtin_type = Reg(io.inner.acquire.bits.payload.builtin_type.clone) - val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone) - val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone) - val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone) - val xact_addr_beat = Reg(io.inner.acquire.bits.payload.addr_beat.clone) - val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone) - val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } - val xact = Acquire( - builtin_type = xact_builtin_type, - a_type = xact_a_type, - client_xact_id = xact_client_xact_id, - addr_block = xact_addr_block, - addr_beat = xact_addr_beat, - data = UInt(0), - subblock = xact_subblock) +class HierarchicalTLIO extends HasInnerTLIO with HasCachedOuterTLIO - val collect_inner_data = Reg(init=Bool(false)) - //TODO: Assert that if xact.uncached, xact_a_type is ReadBlock or WriteBlock - val (inner_data_cnt, inner_data_done) = - Counter(io.inner.acquire.fire() && cacq.payload.hasMultibeatData(), tlDataBeats) - val (outer_data_cnt, outer_data_done) = - Counter(io.outer.acquire.fire() && macq.payload.hasMultibeatData(), tlDataBeats) - val (cgnt_data_cnt, cgnt_data_done) = - Counter(io.inner.grant.fire() && cgnt.payload.hasMultibeatData(), tlDataBeats) +abstract class HierarchicalCoherenceAgent extends CoherenceAgent { + val io = new HierarchicalTLIO + def innerTL = io.inner + def outerTL = io.outer + def incoherent = io.incoherent +} +trait HasTrackerConflictIO extends Bundle { + val has_acquire_conflict = Bool(OUTPUT) + val has_acquire_match = Bool(OUTPUT) + val has_release_match = Bool(OUTPUT) +} - val release_count = Reg(init=UInt(0, width = log2Up(nClients))) - val probe_flags = Reg(init=Bits(0, width = nClients)) - val curr_p_id = PriorityEncoder(probe_flags) +class ManagerXactTrackerIO extends ManagerTLIO with HasTrackerConflictIO +class HierarchicalXactTrackerIO extends HierarchicalTLIO with HasTrackerConflictIO - val pending_outer_write = xact.hasData() - val pending_outer_read = co.requiresOuterRead(xact, co.managerMetadataOnFlush) - - val probe_initial_flags = Bits(width = nClients) - probe_initial_flags := Bits(0) - // issue self-probes for uncached read xacts to facilitate I$ coherence - val probe_self = io.inner.acquire.bits.payload.requiresSelfProbe() - val myflag = Mux(probe_self, Bits(0), UIntToOH(cacq.header.src(log2Up(nClients)-1,0))) - probe_initial_flags := ~(io.tile_incoherent | myflag) - - io.has_acquire_conflict := xact.conflicts(cacq.payload) && - !collect_inner_data && - (state != s_idle) - io.has_release_match := xact.conflicts(crel.payload) && - !crel.payload.isVoluntary() && - (state != s_idle) - - val outer_write_acq = Bundle(UncachedWriteBlock( - client_xact_id = UInt(trackerId), - addr_block = xact_addr_block, - addr_beat = outer_data_cnt, - data = xact_data(outer_data_cnt)), - { case TLId => outerId }) - val outer_write_rel = Bundle(UncachedWriteBlock( - client_xact_id = UInt(trackerId), - addr_block = xact_addr_block, - addr_beat = crel.payload.addr_beat, - data = crel.payload.data), - { case TLId => outerId }) - val outer_read = Bundle(UncachedReadBlock( - client_xact_id = UInt(trackerId), - addr_block = xact_addr_block), - { case TLId => outerId }) - - io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits.payload := outer_read //default - io.outer.grant.ready := io.inner.grant.ready - - io.inner.probe.valid := Bool(false) - io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := xact.makeProbe() - - io.inner.grant.valid := Bool(false) - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := xact.makeGrant(UInt(trackerId)) // Data bypassed in parent - - io.inner.acquire.ready := Bool(false) - io.inner.release.ready := Bool(false) - - when(collect_inner_data) { - io.inner.acquire.ready := Bool(true) - when(io.inner.acquire.valid) { - xact_data(inner_data_cnt) := cacq.payload.data - } - when(inner_data_done) { collect_inner_data := Bool(false) } +abstract class XactTracker extends CoherenceAgentModule { + def connectDataBeatCounter[S <: HasTileLinkData : ClassTag](inc: Bool, data: S, beat: UInt) = { + val multi = data.hasMultibeatData() + val (multi_cnt, multi_done) = Counter(inc && multi, data.tlDataBeats) + val cnt = Mux(multi, multi_cnt, beat) + val done = Mux(multi, multi_done, inc) + (cnt, done) } - - switch (state) { - is(s_idle) { - io.inner.acquire.ready := Bool(true) - val needs_outer_write = cacq.payload.hasData() - val needs_outer_read = co.requiresOuterRead(cacq.payload, co.managerMetadataOnFlush) - when(io.inner.acquire.valid) { - xact_builtin_type := cacq.payload.builtin_type - xact_a_type := cacq.payload.a_type - xact_addr_block := cacq.payload.addr_block - xact_addr_beat := cacq.payload.addr_beat - xact_client_xact_id := cacq.payload.client_xact_id - xact_data(UInt(0)) := cacq.payload.data - xact_subblock := cacq.payload.subblock - xact_src := cacq.header.src - collect_inner_data := cacq.payload.hasMultibeatData() - probe_flags := probe_initial_flags - release_count := PopCount(probe_initial_flags) - state := Mux(probe_initial_flags.orR, s_probe, - Mux(needs_outer_write, s_mem_write, - Mux(needs_outer_read, s_mem_read, s_make_grant))) - } - } - is(s_probe) { - // Generate probes - io.inner.probe.valid := probe_flags.orR - when(io.inner.probe.ready) { - probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) - } - - // Handle releases, which may have data to be written back - io.inner.release.ready := !crel.payload.hasData() || io.outer.acquire.ready - when(io.inner.release.valid) { - when(crel.payload.hasData()) { - io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_write_rel - when(io.outer.acquire.ready) { - when(outer_data_done) { - release_count := release_count - UInt(1) - when(release_count === UInt(1)) { - state := Mux(pending_outer_write, s_mem_write, - Mux(pending_outer_read, s_mem_read, s_make_grant)) - } - } - } - } .otherwise { - release_count := release_count - UInt(1) - when(release_count === UInt(1)) { - state := Mux(pending_outer_write, s_mem_write, - Mux(pending_outer_read, s_mem_read, s_make_grant)) - } - } - } - } - is(s_mem_read) { // Read data from outer memory (possibly what was just written) - io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_read - when(io.outer.acquire.ready) { state := s_mem_resp } - } - is(s_mem_write) { // Write data to outer memory - io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) - else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) - io.outer.acquire.bits.payload := outer_write_acq - when(outer_data_done) { - state := Mux(pending_outer_read, s_mem_read, s_make_grant) - } - } - is(s_make_grant) { // Manufactor a local grant (some kind of permission upgrade) - io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { - state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) - } - } - is(s_mem_resp) { // Wait to forward grants from outer memory - when(io.outer.grant.valid && mgnt.payload.client_xact_id === UInt(trackerId)) { - io.inner.grant.valid := Bool(true) - } - when(cgnt_data_done) { - state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) - } - } - is(s_ack) { // Wait for transaction to complete - when(io.inner.finish.valid && cfin.payload.manager_xact_id === UInt(trackerId)) { - state := s_idle - } - } + def connectOutgoingDataBeatCounter[T <: HasTileLinkData : ClassTag]( + in: DecoupledIO[LogicalNetworkIO[T]], + beat: UInt = UInt(0)) = { + connectDataBeatCounter(in.fire(), in.bits.payload, beat) + } + def connectIncomingDataBeatCounter[T <: HasTileLinkData : ClassTag](in: DecoupledIO[LogicalNetworkIO[T]]) = { + connectDataBeatCounter(in.fire(), in.bits.payload, UInt(0))._2 } } From 8e41fcf6fc58f5194f70fbbf6ed057c9972b7340 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 5 Mar 2015 13:13:12 -0800 Subject: [PATCH 289/374] reduce MemIFTag size, enable non pow2 HellaFLowQueue size --- uncore/src/main/scala/memserdes.scala | 68 ++++++++++++--------------- 1 file changed, 30 insertions(+), 38 deletions(-) diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index f10b3257..7542815b 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -244,20 +244,13 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { (io.tl.release.fire() && rel_has_data), tlDataBeats) val tl_done_out = Reg(init=Bool(false)) val make_grant_ack = Reg(init=Bool(false)) - val grant_for_rel = Grant( - is_builtin_type = Bool(true), - g_type = Grant.voluntaryAckType, - client_xact_id = tag_out, - manager_xact_id = UInt(0)) - val grant_for_acq_write = ManagerMetadata.onReset.makeGrant( - acq = Acquire( - is_builtin_type = tag_out(0), - a_type = tag_out >> UInt(1), - client_xact_id = tag_out >> UInt(io.tl.tlAcquireTypeBits+1), - addr_block = UInt(0)), //DNC - manager_xact_id = UInt(0)) + gnt_arb.io.in(1).valid := Bool(false) - gnt_arb.io.in(1).bits.payload := Mux(data_from_rel, grant_for_rel, grant_for_acq_write) + gnt_arb.io.in(1).bits.payload := Grant( + is_builtin_type = Bool(true), + g_type = Mux(data_from_rel, Grant.voluntaryAckType, Grant.ackType), + client_xact_id = tag_out >> UInt(1), + manager_xact_id = UInt(0)) if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) { val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth)) @@ -278,7 +271,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { when(io.tl.release.valid) { active_out := Bool(true) cmd_sent_out := Bool(false) - tag_out := io.tl.release.bits.payload.client_xact_id + tag_out := Cat(io.tl.release.bits.payload.client_xact_id, + io.tl.release.bits.payload.isVoluntary()) addr_out := io.tl.release.bits.payload.addr_block has_data := rel_has_data data_from_rel := Bool(true) @@ -289,8 +283,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { active_out := Bool(true) cmd_sent_out := Bool(false) tag_out := Cat(io.tl.acquire.bits.payload.client_xact_id, - io.tl.acquire.bits.payload.a_type, - io.tl.acquire.bits.payload.is_builtin_type) + io.tl.acquire.bits.payload.isBuiltInType()) addr_out := io.tl.acquire.bits.payload.addr_block has_data := acq_has_data data_from_rel := Bool(false) @@ -358,7 +351,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { data_from_rel := Bool(true) make_grant_ack := Bool(true) io.mem.req_cmd.bits.rw := rel_has_data - io.mem.req_cmd.bits.tag := io.tl.release.bits.payload.client_xact_id + io.mem.req_cmd.bits.tag := Cat(io.tl.release.bits.payload.client_xact_id, + io.tl.release.bits.payload.isVoluntary()) io.mem.req_cmd.bits.addr := io.tl.release.bits.payload.addr_block io.mem.req_data.bits.data := io.tl.release.bits.payload.data } .elsewhen(io.tl.acquire.valid) { @@ -366,8 +360,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { make_grant_ack := acq_has_data io.mem.req_cmd.bits.rw := acq_has_data io.mem.req_cmd.bits.tag := Cat(io.tl.acquire.bits.payload.client_xact_id, - io.tl.acquire.bits.payload.a_type, - io.tl.acquire.bits.payload.is_builtin_type) + io.tl.acquire.bits.payload.isBuiltInType()) io.mem.req_cmd.bits.addr := io.tl.acquire.bits.payload.addr_block io.mem.req_data.bits.data := io.tl.acquire.bits.payload.data } @@ -409,12 +402,12 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { tl_buf_in := tl_buf_in.fromBits(mif_buf_in.toBits) val tl_prog_in = (tl_cnt_in+UInt(1, width = log2Up(tlDataBeats+1)))*UInt(tlDataBits) val mif_prog_in = mif_cnt_in*UInt(mifDataBits) - gnt_arb.io.in(0).bits.payload := ManagerMetadata.onReset.makeGrant( - acq = Acquire( - is_builtin_type = tag_in(0), - a_type = tag_in >> UInt(1), - client_xact_id = tag_in >> UInt(io.tl.tlAcquireTypeBits+1), - addr_block = UInt(0)), //DNC + gnt_arb.io.in(0).bits.payload := Grant( + is_builtin_type = tag_in(0), + g_type = Mux(tag_in(0), + Grant.dataBlockType, + UInt(0)), // TODO: Assumes MI or MEI protocol + client_xact_id = tag_in >> UInt(1), manager_xact_id = UInt(0), addr_beat = tl_cnt_in, data = tl_buf_in(tl_cnt_in)) @@ -442,23 +435,22 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { } else { // Don't generate all the uneeded data buffers and flow resp gnt_arb.io.in(0).valid := io.mem.resp.valid io.mem.resp.ready := gnt_arb.io.in(0).ready - gnt_arb.io.in(0).bits.payload := - ManagerMetadata.onReset.makeGrant( - acq = Acquire( - is_builtin_type = io.mem.resp.bits.tag(0), - a_type = io.mem.resp.bits.tag >> UInt(1), - client_xact_id = io.mem.resp.bits.tag >> UInt(io.tl.tlAcquireTypeBits+1), - addr_block = UInt(0)), //DNC - manager_xact_id = UInt(0), - addr_beat = tl_cnt_in, - data = io.mem.resp.bits.data) + gnt_arb.io.in(0).bits.payload := Grant( + is_builtin_type = io.mem.resp.bits.tag(0), + g_type = Mux(io.mem.resp.bits.tag(0), + Grant.dataBlockType, + UInt(0)), // TODO: Assumes MI or MEI protocol + client_xact_id = io.mem.resp.bits.tag >> UInt(1), + manager_xact_id = UInt(0), + addr_beat = tl_cnt_in, + data = io.mem.resp.bits.data) } } class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module { val io = new QueueIO(data, entries) - require(isPow2(entries) && entries > 1) + require(entries > 1) val do_flow = Bool() val do_enq = io.enq.fire() && !do_flow @@ -466,7 +458,7 @@ class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module val maybe_full = Reg(init=Bool(false)) val enq_ptr = Counter(do_enq, entries)._1 - val deq_ptr = Counter(do_deq, entries)._1 + val (deq_ptr, deq_done) = Counter(do_deq, entries) when (do_enq != do_deq) { maybe_full := do_enq } val ptr_match = enq_ptr === deq_ptr @@ -482,7 +474,7 @@ class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module when (do_enq) { ram(enq_ptr) := io.enq.bits } when (io.deq.ready && (atLeastTwo || !io.deq.valid && !empty)) { ram_out_valid := Bool(true) - ram_addr := Mux(io.deq.valid, deq_ptr + UInt(1), deq_ptr) + ram_addr := Mux(io.deq.valid, Mux(deq_done, UInt(0), deq_ptr + UInt(1)), deq_ptr) } io.deq.valid := Mux(empty, io.enq.valid, ram_out_valid) From df79e7ff8dcadc09d91d6b2dede2b583487f2af0 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 5 Mar 2015 14:40:31 -0800 Subject: [PATCH 290/374] secondary miss bug --- uncore/src/main/scala/coherence.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 2524f3d1..1e8d1b60 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -60,9 +60,7 @@ trait HasClientSideCoherencePolicy { first_cmd: UInt, second_cmd: UInt, meta: ClientMetadata): Bool = { - isWriteIntent(second_cmd) && - !isWriteIntent(first_cmd) || - (getAcquireType(first_cmd, meta) != getAcquireType(second_cmd, meta)) + isWriteIntent(second_cmd) && !isWriteIntent(first_cmd) } //TODO: Assumes all cache ctrl ops writeback dirty data, and // doesn't issue transaction when e.g. downgrading Exclusive to Shared: From 002f1a1b393df17b4fe0e07f0a43dc36bdc0ac4f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 9 Mar 2015 12:40:37 -0700 Subject: [PATCH 291/374] pin outer finish header --- uncore/src/main/scala/cache.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 75eb897d..aa2097af 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -831,6 +831,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.ognt().requiresAck()) { pending_finish.payload := pending_finish_on_ognt pending_finish.header.dst := io.outer.grant.bits.header.src + pending_finish.header.src := UInt(bankId) state := s_outer_finish }.otherwise { state := Mux(!do_allocate, s_inner_grant, From a1f04386f75664654a049c1d5b4078f402515c70 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 9 Mar 2015 16:34:59 -0700 Subject: [PATCH 292/374] Headerless TileLinkIO and arbiters --- uncore/src/main/scala/htif.scala | 7 +- uncore/src/main/scala/tilelink.scala | 123 ++++++++++++++++++++------- 2 files changed, 92 insertions(+), 38 deletions(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 4d5e1135..644a8317 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -60,7 +60,7 @@ class SCRIO extends HTIFBundle { class HTIFModuleIO extends HTIFBundle { val host = new HostIO val cpu = Vec.fill(nCores){new HTIFIO}.flip - val mem = new TileLinkIO + val mem = new HeaderlessUncachedTileLinkIO val scr = new SCRIO } @@ -195,19 +195,16 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { val init_addr = addr.toUInt >> UInt(offsetBits-3) io.mem.acquire.valid := state === state_mem_rreq || state === state_mem_wreq - io.mem.acquire.bits.payload := Mux(cmd === cmd_writemem, + io.mem.acquire.bits := Mux(cmd === cmd_writemem, PutBlock( addr_block = init_addr, addr_beat = cnt, client_xact_id = UInt(0), data = mem_req_data), GetBlock(addr_block = init_addr)) - io.mem.acquire.bits.payload.data := mem_req_data io.mem.finish.valid := (state === state_mem_finish) && mem_needs_ack io.mem.finish.bits.payload.manager_xact_id := mem_gxid io.mem.finish.bits.header.dst := mem_gsrc - io.mem.probe.ready := Bool(false) - io.mem.release.valid := Bool(false) val pcr_reset = UInt(pcr_RESET)(pcr_addr.getWidth-1,0) val pcrReadData = Reg(Bits(width = io.cpu(0).pcr_rep.bits.getWidth)) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index c8996ddf..3d46ae80 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -3,6 +3,7 @@ package uncore import Chisel._ import scala.math.max +import scala.reflect.ClassTag // Parameters exposed to the top-level design, set based on // external requirements or design space exploration @@ -290,7 +291,6 @@ object Probe { } } - class Release extends ClientToManagerChannel with HasCacheBlockAddress with HasClientTransactionId @@ -420,6 +420,42 @@ object TileLinkIOWrapper { def apply(tl: TileLinkIO) = tl } +// This version of TileLinkIO does not contain network headers for packets +// that originate in the Clients (i.e. Acquire and Release). These headers +// are provided in the top-level that instantiates the clients and network. +// By eliding the header subbundles within the clients we can enable +// hierarchical P&R while minimizing unconnected port errors in GDS. +class HeaderlessUncachedTileLinkIO extends TLBundle { + val acquire = new DecoupledIO(new Acquire) + val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip + val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) +} + +class HeaderlessTileLinkIO extends HeaderlessUncachedTileLinkIO { + val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip + val release = new DecoupledIO(new Release) +} + +class HeaderlessTileLinkIOWrapper extends TLModule { + val io = new Bundle { + val in = new HeaderlessUncachedTileLinkIO().flip + val out = new HeaderlessTileLinkIO + } + io.out.acquire <> io.in.acquire + io.out.grant <> io.in.grant + io.out.finish <> io.in.finish + io.out.probe.ready := Bool(false) + io.out.release.valid := Bool(false) +} + +object HeaderlessTileLinkIOWrapper { + def apply(utl: HeaderlessUncachedTileLinkIO): HeaderlessTileLinkIO = { + val conv = Module(new HeaderlessTileLinkIOWrapper) + conv.io.in <> utl + conv.io.out + } +} + abstract trait HasArbiterTypes { val arbN: Int type ManagerSourcedWithId = ManagerToClientChannel with HasClientTransactionId @@ -428,6 +464,7 @@ abstract trait HasArbiterTypes { HasClientTransactionId with HasTileLinkData } + // Utility functions for constructing TileLinkIO arbiters abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule with HasArbiterTypes { @@ -438,34 +475,65 @@ abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule def managerSourcedClientXactId(in: ManagerSourcedWithId): Bits def arbIdx(in: ManagerSourcedWithId): UInt - def hookupClientSource[M <: ClientSourcedWithIdAndData] - (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], - out: DecoupledIO[LogicalNetworkIO[M]]) { + def hookupClientSource[M <: ClientSourcedWithIdAndData : ClassTag] + (clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], + mngr: DecoupledIO[LogicalNetworkIO[M]]) { def hasData(m: LogicalNetworkIO[M]) = m.payload.hasMultibeatData() - val arb = Module(new LockingRRArbiter(out.bits.clone, arbN, params(TLDataBeats), Some(hasData _))) - out <> arb.io.out - ins.zipWithIndex.zip(arb.io.in).map{ case ((req,id), arb) => { + val arb = Module(new LockingRRArbiter(mngr.bits.clone, arbN, params(TLDataBeats), Some(hasData _))) + clts.zipWithIndex.zip(arb.io.in).map{ case ((req, id), arb) => { arb.valid := req.valid arb.bits := req.bits arb.bits.payload.client_xact_id := clientSourcedClientXactId(req.bits.payload, id) req.ready := arb.ready }} + arb.io.out <> mngr } - def hookupManagerSource[M <: ManagerSourcedWithId] - (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], - out: DecoupledIO[LogicalNetworkIO[M]]) { - out.ready := Bool(false) + def hookupClientSourceHeaderless[M <: ClientSourcedWithIdAndData : ClassTag] + (clts: Seq[DecoupledIO[M]], + mngr: DecoupledIO[M]) { + def hasData(m: M) = m.hasMultibeatData() + val arb = Module(new LockingRRArbiter(mngr.bits.clone, arbN, params(TLDataBeats), Some(hasData _))) + clts.zipWithIndex.zip(arb.io.in).map{ case ((req, id), arb) => { + arb.valid := req.valid + arb.bits := req.bits + arb.bits.client_xact_id := clientSourcedClientXactId(req.bits, id) + req.ready := arb.ready + }} + arb.io.out <> mngr + } + + def hookupFinish[M <: LogicalNetworkIO[Finish] : ClassTag] + (clts: Seq[DecoupledIO[M]], + mngr: DecoupledIO[M]) { + val arb = Module(new RRArbiter(mngr.bits.clone, arbN)) + arb.io.in zip clts map { case (arb, req) => arb <> req } + arb.io.out <> mngr + } + + def hookupManagerSourceWithId[M <: ManagerSourcedWithId] + (clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], + mngr: DecoupledIO[LogicalNetworkIO[M]]) { + mngr.ready := Bool(false) for (i <- 0 until arbN) { - ins(i).valid := Bool(false) - when (arbIdx(out.bits.payload) === UInt(i)) { - ins(i).valid := out.valid - out.ready := ins(i).ready + clts(i).valid := Bool(false) + when (arbIdx(mngr.bits.payload) === UInt(i)) { + clts(i).valid := mngr.valid + mngr.ready := clts(i).ready } - ins(i).bits := out.bits - ins(i).bits.payload.client_xact_id := managerSourcedClientXactId(out.bits.payload) + clts(i).bits := mngr.bits + clts(i).bits.payload.client_xact_id := + managerSourcedClientXactId(mngr.bits.payload) } } + + def hookupManagerSourceBroadcast[M <: ManagerToClientChannel] + (clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], + mngr: DecoupledIO[LogicalNetworkIO[M]]) { + clts.map{ _.valid := mngr.valid } + clts.map{ _.bits := mngr.bits } + mngr.ready := clts.map(_.ready).reduce(_||_) + } } abstract class UncachedTileLinkIOArbiter(n: Int) @@ -474,13 +542,9 @@ abstract class UncachedTileLinkIOArbiter(n: Int) val in = Vec.fill(n){new UncachedTileLinkIO}.flip val out = new UncachedTileLinkIO } - hookupClientSource(io.in.map(_.acquire), io.out.acquire) - hookupManagerSource(io.in.map(_.grant), io.out.grant) - - val finish_arb = Module(new RRArbiter(new LogicalNetworkIO(new Finish), n)) - io.out.finish <> finish_arb.io.out - finish_arb.io.in zip io.in map { case (arb, req) => arb <> req.finish } + hookupFinish(io.in.map(_.finish), io.out.finish) + hookupManagerSourceWithId(io.in.map(_.grant), io.out.grant) } abstract class TileLinkIOArbiter(n: Int) extends TileLinkArbiterLike(n) { @@ -488,18 +552,11 @@ abstract class TileLinkIOArbiter(n: Int) extends TileLinkArbiterLike(n) { val in = Vec.fill(n){new TileLinkIO}.flip val out = new TileLinkIO } - hookupClientSource(io.in.map(_.acquire), io.out.acquire) hookupClientSource(io.in.map(_.release), io.out.release) - hookupManagerSource(io.in.map(_.grant), io.out.grant) - - io.in.map{ _.probe.valid := io.out.probe.valid } - io.in.map{ _.probe.bits := io.out.probe.bits } - io.out.probe.ready := io.in.map(_.probe.ready).reduce(_||_) - - val finish_arb = Module(new RRArbiter(new LogicalNetworkIO(new Finish), n)) - io.out.finish <> finish_arb.io.out - finish_arb.io.in zip io.in map { case (arb, req) => arb <> req.finish } + hookupFinish(io.in.map(_.finish), io.out.finish) + hookupManagerSourceBroadcast(io.in.map(_.probe), io.out.probe) + hookupManagerSourceWithId(io.in.map(_.grant), io.out.grant) } // Appends the port index of the arbiter to the client_xact_id From 17072a00416470e93e0b9ffc817f89b950753c23 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 10 Mar 2015 01:15:03 -0700 Subject: [PATCH 293/374] L2 Writeback bugfix --- uncore/src/main/scala/cache.scala | 27 +++++++++++-------- uncore/src/main/scala/coherence.scala | 2 ++ uncore/src/main/scala/memserdes.scala | 37 +++++++++++++++++---------- 3 files changed, 42 insertions(+), 24 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index aa2097af..ec25501c 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -566,17 +566,16 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val is_hit = xact_tag_match && xact_meta.coh.outer.isHit(xact.op_code()) val do_allocate = xact.allocate() val needs_writeback = !xact_tag_match && do_allocate && - xact_meta.coh.outer.requiresVoluntaryWriteback() - val needs_probes = xact_meta.coh.inner.requiresProbes(xact) + (xact_meta.coh.outer.requiresVoluntaryWriteback() || + xact_meta.coh.inner.requiresProbesOnVoluntaryWriteback()) val pending_coh_on_hit = HierarchicalMetadata( io.meta.resp.bits.meta.coh.inner, io.meta.resp.bits.meta.coh.outer.onHit(xact.op_code())) - val pending_coh_on_irel = HierarchicalMetadata( - pending_coh.inner.onRelease( + val pending_icoh_on_irel = pending_coh.inner.onRelease( incoming = io.irel(), - src = io.inner.release.bits.header.src), - pending_coh.outer.onHit(M_XWR)) // WB is a write + src = io.inner.release.bits.header.src) + val pending_ocoh_on_irel = pending_coh.outer.onHit(M_XWR) // WB is a write val pending_coh_on_ognt = HierarchicalMetadata( ManagerMetadata.onReset, pending_coh.outer.onGrant(io.ognt(), xact.op_code())) @@ -768,7 +767,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val _tag_match = io.meta.resp.bits.tag_match val _is_hit = _tag_match && _coh.outer.isHit(xact.op_code()) val _needs_writeback = !_tag_match && do_allocate && - _coh.outer.requiresVoluntaryWriteback() + (_coh.outer.requiresVoluntaryWriteback() || + _coh.inner.requiresProbesOnVoluntaryWriteback()) val _needs_probes = _tag_match && _coh.inner.requiresProbes(xact) when(_is_hit) { pending_coh := pending_coh_on_hit } when(_needs_probes) { @@ -796,12 +796,13 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) when(io.inner.release.valid) { - pending_coh := pending_coh_on_irel + pending_coh.inner := pending_icoh_on_irel // Handle released dirty data //TODO: make sure cacq data is actually present before accpeting // release data to merge! when(io.irel().hasData()) { irel_had_data := Bool(true) + pending_coh.outer := pending_ocoh_on_irel mergeDataInner(data_buffer, io.irel()) } // We don't decrement release_count until we've received all the data beats. @@ -931,8 +932,9 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val resp_data_done = connectInternalDataBeatCounter(io.data.resp) val pending_icoh_on_irel = xact_coh.inner.onRelease( - incoming = io.irel(), + incoming = io.irel(), src = io.inner.release.bits.header.src) + val pending_ocoh_on_irel = xact_coh.outer.onHit(M_XWR) // WB is a write io.has_acquire_conflict := Bool(false) io.has_acquire_match := Bool(false) @@ -1008,6 +1010,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { // Handle released dirty data when(io.irel().hasData()) { irel_had_data := Bool(true) + xact_coh.outer := pending_ocoh_on_irel data_buffer(io.irel().addr_beat) := io.irel().data } // We don't decrement release_count until we've received all the data beats. @@ -1016,7 +1019,11 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { } } when(release_count === UInt(0)) { - state := Mux(irel_had_data, s_outer_release, s_data_read) + state := Mux(irel_had_data, // If someone released a dirty block + s_outer_release, // write that block back, otherwise + Mux(xact_coh.outer.requiresVoluntaryWriteback(), + s_data_read, // write extant dirty data back, or just + s_wb_resp)) // drop a clean block after collecting acks } } is(s_data_read) { diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 1e8d1b60..862eb484 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -383,6 +383,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { val nManagerStates = 0 // TODO: We could add a Shared state to avoid probing // only a single sharer (also would need // notification msg to track clean drops) + // Also could avoid probes on outer WBs. def requiresProbes(a: Acquire, meta: ManagerMetadata) = Mux(dir.none(meta.sharers), Bool(false), @@ -501,6 +502,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { val nManagerStates = 0 // TODO: We could add a Shared state to avoid probing // only a single sharer (also would need // notification msg to track clean drops) + // Also could avoid probes on outer WBs. def requiresProbes(a: Acquire, meta: ManagerMetadata) = Mux(dir.none(meta.sharers), Bool(false), diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 7542815b..eafc44bd 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -212,6 +212,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { val mem = new MemIO } val mifTagBits = params(MIFTagBits) + val mifAddrBits = params(MIFAddrBits) val mifDataBits = params(MIFDataBits) val mifDataBeats = params(MIFDataBeats) val tlDataBits = params(TLDataBits) @@ -235,8 +236,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { // Decompose outgoing TL Acquires into MemIO cmd and data val active_out = Reg(init=Bool(false)) val cmd_sent_out = Reg(init=Bool(false)) - val tag_out = Reg(Bits()) - val addr_out = Reg(Bits()) + val tag_out = Reg(UInt(width = mifTagBits)) + val addr_out = Reg(UInt(width = mifAddrBits)) val has_data = Reg(init=Bool(false)) val data_from_rel = Reg(init=Bool(false)) val (tl_cnt_out, tl_wrap_out) = @@ -343,26 +344,34 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { active_out := !io.mem.req_cmd.ready || io.mem.req_data.valid io.mem.req_cmd.valid := Bool(true) cmd_sent_out := io.mem.req_cmd.ready - tag_out := io.mem.req_cmd.bits.tag - addr_out := io.mem.req_data.bits.data - has_data := io.mem.req_cmd.bits.rw tl_done_out := tl_wrap_out when(io.tl.release.valid) { data_from_rel := Bool(true) make_grant_ack := Bool(true) - io.mem.req_cmd.bits.rw := rel_has_data - io.mem.req_cmd.bits.tag := Cat(io.tl.release.bits.payload.client_xact_id, - io.tl.release.bits.payload.isVoluntary()) - io.mem.req_cmd.bits.addr := io.tl.release.bits.payload.addr_block io.mem.req_data.bits.data := io.tl.release.bits.payload.data + val tag = Cat(io.tl.release.bits.payload.client_xact_id, + io.tl.release.bits.payload.isVoluntary()) + val addr = io.tl.release.bits.payload.addr_block + io.mem.req_cmd.bits.tag := tag + io.mem.req_cmd.bits.addr := addr + io.mem.req_cmd.bits.rw := rel_has_data + tag_out := tag + addr_out := addr + has_data := rel_has_data } .elsewhen(io.tl.acquire.valid) { data_from_rel := Bool(false) make_grant_ack := acq_has_data - io.mem.req_cmd.bits.rw := acq_has_data - io.mem.req_cmd.bits.tag := Cat(io.tl.acquire.bits.payload.client_xact_id, - io.tl.acquire.bits.payload.isBuiltInType()) - io.mem.req_cmd.bits.addr := io.tl.acquire.bits.payload.addr_block io.mem.req_data.bits.data := io.tl.acquire.bits.payload.data + io.mem.req_cmd.bits.rw := acq_has_data + val tag = Cat(io.tl.acquire.bits.payload.client_xact_id, + io.tl.acquire.bits.payload.isBuiltInType()) + val addr = io.tl.acquire.bits.payload.addr_block + io.mem.req_cmd.bits.tag := tag + io.mem.req_cmd.bits.addr := addr + io.mem.req_cmd.bits.rw := acq_has_data + tag_out := tag + addr_out := addr + has_data := acq_has_data } } } @@ -380,7 +389,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { } when(tl_wrap_out) { tl_done_out := Bool(true) } when(tl_done_out && make_grant_ack) { - gnt_arb.io.in(1).valid := Bool(true) + gnt_arb.io.in(1).valid := Bool(true) // TODO: grants for voluntary acks? when(gnt_arb.io.in(1).ready) { make_grant_ack := Bool(false) } } when(cmd_sent_out && (!has_data || tl_done_out) && !make_grant_ack) { From 3ab1aca7decddfa7e1bf5ee35c864c43076e23ba Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Wed, 11 Mar 2015 01:56:47 -0700 Subject: [PATCH 294/374] L2 subblock access bugfix --- uncore/src/main/scala/cache.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index ec25501c..1d117ded 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -412,7 +412,7 @@ class L2XactTrackerIO extends HierarchicalXactTrackerIO { abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { def connectDataBeatCounter[S <: L2HellaCacheBundle](inc: Bool, data: S, beat: UInt, full_block: Bool) = { if(data.refillCycles > 1) { - val (multi_cnt, multi_done) = Counter(inc, data.refillCycles) + val (multi_cnt, multi_done) = Counter(full_block && inc, data.refillCycles) (Mux(!full_block, beat, multi_cnt), Mux(!full_block, inc, multi_done)) } else { (UInt(0), inc) } } From b4ed1d912160964c5ea41a1b2614afa21c83f71f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 11 Mar 2015 14:28:17 -0700 Subject: [PATCH 295/374] Add builtin prefetch types to TileLink --- uncore/src/main/scala/tilelink.scala | 48 ++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 3d46ae80..ef0d3307 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -109,6 +109,8 @@ class Acquire extends ClientToManagerChannel def isSubBlockType(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesOnSubBlocks.contains(a_type) + def isPrefetch(dummy: Int = 0): Bool = isBuiltInType() && is(Acquire.prefetchType) + // Assumes no custom types have data def hasData(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesWithData.contains(a_type) @@ -124,7 +126,8 @@ class Acquire extends ClientToManagerChannel Acquire.getBlockType -> Grant.dataBlockType, Acquire.putType -> Grant.ackType, Acquire.putBlockType -> Grant.ackType, - Acquire.putAtomicType -> Grant.dataBeatType)) + Acquire.putAtomicType -> Grant.dataBeatType, + Acquire.prefetchType -> Grant.ackType)) } } @@ -136,6 +139,7 @@ object Acquire { def putType = UInt("b010") def putBlockType = UInt("b011") def putAtomicType = UInt("b100") + def prefetchType = UInt("b101") def typesWithData = Vec(putType, putBlockType, putAtomicType) def typesWithMultibeatData = Vec(putBlockType) def typesOnSubBlocks = Vec(putType, getType, putAtomicType) @@ -172,9 +176,9 @@ object Acquire { // Asks for a single TileLink beat of data object Get { def apply( - client_xact_id: UInt, - addr_block: UInt, - addr_beat: UInt, + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, alloc: Bool = Bool(true)): Acquire = { Acquire( is_builtin_type = Bool(true), @@ -189,8 +193,8 @@ object Get { // Asks for an entire cache block of data object GetBlock { def apply( - client_xact_id: UInt = UInt(0), - addr_block: UInt, + client_xact_id: UInt = UInt(0), + addr_block: UInt, alloc: Bool = Bool(true)): Acquire = { Acquire( is_builtin_type = Bool(true), @@ -201,6 +205,22 @@ object GetBlock { } } +// Prefetch a cache block into the next level of the memory hierarchy +// with read permissions +object GetPrefetch { + def apply( + client_xact_id: UInt, + addr_block: UInt): Acquire = { + Acquire( + is_builtin_type = Bool(true), + a_type = Acquire.prefetchType, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = UInt(0), + union = Cat(M_XRD, Bool(true))) + } +} + // Writes up to a single TileLink beat of data, using mask object Put { def apply( @@ -275,6 +295,22 @@ object PutAtomic { } } +// Prefetch a cache block into the next level of the memory hierarchy +// with write permissions +object PutPrefetch { + def apply( + client_xact_id: UInt, + addr_block: UInt): Acquire = { + Acquire( + is_builtin_type = Bool(true), + a_type = Acquire.prefetchType, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = UInt(0), + union = Cat(M_XWR, Bool(true))) + } +} + class Probe extends ManagerToClientChannel with HasCacheBlockAddress { val p_type = UInt(width = tlCoh.probeTypeWidth) From 059575c334dd34cc818231fdd5ff82769171c28f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 11 Mar 2015 15:43:41 -0700 Subject: [PATCH 296/374] cleanup mergeData and prep for cleaner data_buffer in L2 --- uncore/src/main/scala/cache.scala | 72 +++++++++++++++--------------- uncore/src/main/scala/uncore.scala | 2 + 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 1d117ded..87473587 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -6,15 +6,16 @@ import Chisel._ case object CacheName extends Field[String] case object NSets extends Field[Int] case object NWays extends Field[Int] -case object BlockOffBits extends Field[Int] case object RowBits extends Field[Int] case object Replacer extends Field[() => ReplacementPolicy] case object AmoAluOperandBits extends Field[Int] case object L2DirectoryRepresentation extends Field[DirectoryRepresentation] +case object CacheBlockBytes extends Field[Int] +case object CacheBlockOffsetBits extends Field[Int] abstract trait CacheParameters extends UsesParameters { val nSets = params(NSets) - val blockOffBits = params(BlockOffBits) + val blockOffBits = params(CacheBlockOffsetBits) val idxBits = log2Up(nSets) val untagBits = blockOffBits + idxBits val tagBits = params(PAddrBits) - untagBits @@ -165,6 +166,7 @@ abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgen val blockAddrBits = params(TLBlockAddrBits) val refillCyclesPerBeat = outerDataBits/rowBits val refillCycles = refillCyclesPerBeat*outerDataBeats + val internalDataBeats = params(CacheBlockBytes)*8/rowBits require(refillCyclesPerBeat == 1) val amoAluOperandBits = params(AmoAluOperandBits) require(amoAluOperandBits <= innerDataBits) @@ -410,6 +412,14 @@ class L2XactTrackerIO extends HierarchicalXactTrackerIO { } abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { + class CacheBlockBuffer { + val buffer = Reg(Bits(width = params(CacheBlockBytes)*8)) + + def internal = Vec.fill(internalDataBeats){ Bits(width = rowBits) }.fromBits(buffer) + def inner = Vec.fill(innerDataBeats){ Bits(width = innerDataBits) }.fromBits(buffer) + def outer = Vec.fill(outerDataBeats){ Bits(width = outerDataBits) }.fromBits(buffer) + } + def connectDataBeatCounter[S <: L2HellaCacheBundle](inc: Bool, data: S, beat: UInt, full_block: Bool) = { if(data.refillCycles > 1) { val (multi_cnt, multi_done) = Counter(full_block && inc, data.refillCycles) @@ -554,9 +564,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_src = Reg(io.inner.acquire.bits.header.src.clone) val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId); case TLDataBits => 0 })) - val data_buffer = Vec.fill(innerDataBeats+1) { // Extra entry holds AMO result + val data_buffer = Vec.fill(innerDataBeats) { Reg(io.iacq().data.clone) - } + } + val amo_result = Reg(io.iacq().data.clone) val xact_tag_match = Reg{ Bool() } val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } @@ -612,44 +623,31 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { amoalu.io.cmd := xact.op_code() amoalu.io.typ := xact.op_size() amoalu.io.lhs := io.data.resp.bits.data //default - amoalu.io.rhs := data_buffer(0) // default + amoalu.io.rhs := data_buffer.head // default - // TODO: figure out how to merge the following three versions of this func - def mergeDataInternal[T <: HasL2Data](buffer: Vec[UInt], incoming: T) { - val old_data = incoming.data - val new_data = buffer(incoming.addr_beat) + def mergeData[T <: HasTileLinkData] + (byteAddrBits: Int, dataBits: Int) + (buffer: Vec[UInt], beat: UInt, incoming: UInt) { + val old_data = incoming + val new_data = buffer(beat) val amoOpSz = UInt(amoAluOperandBits) - val offset = xact.addr_byte()(innerByteAddrBits-1, log2Up(amoAluOperandBits/8)) + val offset = xact.addr_byte()(byteAddrBits-1, log2Up(amoAluOperandBits/8)) amoalu.io.lhs := old_data >> offset*amoOpSz amoalu.io.rhs := new_data >> offset*amoOpSz - val wmask = + val valid_beat = xact.addr_beat === beat + val wmask = Fill(dataBits, valid_beat) & Mux(xact.is(Acquire.putAtomicType), FillInterleaved(amoAluOperandBits, UIntToOH(offset)), Mux(xact.is(Acquire.putBlockType) || xact.is(Acquire.putType), FillInterleaved(8, xact.write_mask()), - UInt(0, width = innerDataBits))) - buffer(incoming.addr_beat) := ~wmask & old_data | wmask & + UInt(0, width = dataBits))) + buffer(beat) := ~wmask & old_data | wmask & Mux(xact.is(Acquire.putAtomicType), amoalu.io.out << offset*amoOpSz, new_data) - when(xact.is(Acquire.putAtomicType)) { buffer(innerDataBeats) := old_data } // For AMO result - } - def mergeDataInner[T <: HasTileLinkData](buffer: Vec[UInt], incoming: T) = mergeDataOuter(buffer, incoming) - def mergeDataOuter[T <: HasTileLinkData](buffer: Vec[UInt], incoming: T) { - val old_data = incoming.data - val new_data = buffer(incoming.addr_beat) - val amoOpSz = UInt(amoAluOperandBits) - val offset = xact.addr_byte()(innerByteAddrBits-1, log2Up(amoAluOperandBits/8)) - amoalu.io.lhs := old_data >> offset*amoOpSz - amoalu.io.rhs := new_data >> offset*amoOpSz - val wmask = - Mux(xact.is(Acquire.putAtomicType), - FillInterleaved(amoAluOperandBits, UIntToOH(offset)), - Mux(xact.is(Acquire.putBlockType) || xact.is(Acquire.putType), - FillInterleaved(8, xact.write_mask()), - UInt(0, width = innerDataBits))) - buffer(incoming.addr_beat) := ~wmask & old_data | wmask & - Mux(xact.is(Acquire.putAtomicType), amoalu.io.out << offset*amoOpSz, new_data) - when(xact.is(Acquire.putAtomicType)) { buffer(innerDataBeats) := old_data } // For AMO result + when(xact.is(Acquire.putAtomicType) && valid_beat) { amo_result := old_data } } + val mergeDataInternal = mergeData(log2Up(rowBits/8), rowBits) _ + val mergeDataInner = mergeData(innerByteAddrBits, innerDataBits) _ + val mergeDataOuter = mergeData(outerByteAddrBits, outerDataBits) _ //TODO: Allow hit under miss for stores val in_same_set = xact.addr_block(idxMSB,idxLSB) === @@ -694,7 +692,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { manager_xact_id = UInt(trackerId), addr_beat = ignt_data_cnt, data = Mux(xact.is(Acquire.putAtomicType), - data_buffer(innerDataBeats), + amo_result, data_buffer(ignt_data_cnt))) io.inner.acquire.ready := Bool(false) @@ -803,7 +801,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.irel().hasData()) { irel_had_data := Bool(true) pending_coh.outer := pending_ocoh_on_irel - mergeDataInner(data_buffer, io.irel()) + mergeDataInner(data_buffer, io.irel().addr_beat, io.irel().data) } // We don't decrement release_count until we've received all the data beats. when(!io.irel().hasMultibeatData() || irel_data_done) { @@ -824,7 +822,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.grant.ready := Bool(true) when(io.outer.grant.valid) { when(io.ognt().hasData()) { - mergeDataOuter(data_buffer, io.ognt()) + mergeDataOuter(data_buffer, io.ognt().addr_beat, io.ognt().data) ognt_had_data := Bool(true) } when(ognt_data_done) { @@ -851,13 +849,13 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { is(s_data_read) { io.data.read.valid := !collect_iacq_data || iacq_data_valid(read_data_cnt) when(io.data.resp.valid) { - mergeDataInternal(data_buffer, io.data.resp.bits) + mergeDataInternal(data_buffer, io.data.resp.bits.addr_beat, io.data.resp.bits.data) } when(read_data_done) { state := s_data_resp } } is(s_data_resp) { when(io.data.resp.valid) { - mergeDataInternal(data_buffer, io.data.resp.bits) + mergeDataInternal(data_buffer, io.data.resp.bits.addr_beat, io.data.resp.bits.data) } when(resp_data_done) { state := Mux(xact.hasData(), s_data_write, s_inner_grant) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 3c334376..05319ee9 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -21,6 +21,8 @@ trait CoherenceAgentParameters extends UsesParameters { def outerTLParams = params.alterPartial({ case TLId => params(OuterTLId)}) val outerDataBeats = outerTLParams(TLDataBeats) val outerDataBits = outerTLParams(TLDataBits) + val outerBeatAddrBits = log2Up(outerDataBeats) + val outerByteAddrBits = log2Up(outerDataBits/8) def innerTLParams = params.alterPartial({case TLId => params(InnerTLId)}) val innerDataBeats = innerTLParams(TLDataBeats) val innerDataBits = innerTLParams(TLDataBits) From 1aff919c24a80b54e95efa5f1389da1df7b4f1aa Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 11 Mar 2015 17:32:06 -0700 Subject: [PATCH 297/374] added prefetchAck Grant type --- uncore/src/main/scala/memserdes.scala | 6 +++--- uncore/src/main/scala/tilelink.scala | 29 ++++++++++++++------------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index eafc44bd..b9786b4c 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -249,7 +249,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { gnt_arb.io.in(1).valid := Bool(false) gnt_arb.io.in(1).bits.payload := Grant( is_builtin_type = Bool(true), - g_type = Mux(data_from_rel, Grant.voluntaryAckType, Grant.ackType), + g_type = Mux(data_from_rel, Grant.voluntaryAckType, Grant.putAckType), client_xact_id = tag_out >> UInt(1), manager_xact_id = UInt(0)) @@ -414,7 +414,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { gnt_arb.io.in(0).bits.payload := Grant( is_builtin_type = tag_in(0), g_type = Mux(tag_in(0), - Grant.dataBlockType, + Grant.getDataBlockType, UInt(0)), // TODO: Assumes MI or MEI protocol client_xact_id = tag_in >> UInt(1), manager_xact_id = UInt(0), @@ -447,7 +447,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { gnt_arb.io.in(0).bits.payload := Grant( is_builtin_type = io.mem.resp.bits.tag(0), g_type = Mux(io.mem.resp.bits.tag(0), - Grant.dataBlockType, + Grant.getDataBlockType, UInt(0)), // TODO: Assumes MI or MEI protocol client_xact_id = io.mem.resp.bits.tag >> UInt(1), manager_xact_id = UInt(0), diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index ef0d3307..531d3f52 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -121,13 +121,13 @@ class Acquire extends ClientToManagerChannel def requiresSelfProbe(dummy: Int = 0) = isBuiltInType() && a_type === Acquire.getBlockType def getBuiltInGrantType(dummy: Int = 0): UInt = { - MuxLookup(this.a_type, Grant.ackType, Array( - Acquire.getType -> Grant.dataBeatType, - Acquire.getBlockType -> Grant.dataBlockType, - Acquire.putType -> Grant.ackType, - Acquire.putBlockType -> Grant.ackType, - Acquire.putAtomicType -> Grant.dataBeatType, - Acquire.prefetchType -> Grant.ackType)) + MuxLookup(this.a_type, Grant.putAckType, Array( + Acquire.getType -> Grant.getDataBeatType, + Acquire.getBlockType -> Grant.getDataBlockType, + Acquire.putType -> Grant.putAckType, + Acquire.putBlockType -> Grant.putAckType, + Acquire.putAtomicType -> Grant.getDataBeatType, + Acquire.prefetchType -> Grant.prefetchAckType)) } } @@ -389,13 +389,14 @@ class Grant extends ManagerToClientChannel } object Grant { - val nBuiltInTypes = 4 - def voluntaryAckType = UInt("b00") - def ackType = UInt("b01") - def dataBeatType = UInt("b10") - def dataBlockType = UInt("b11") - def typesWithData = Vec(dataBlockType, dataBeatType) - def typesWithMultibeatData= Vec(dataBlockType) + val nBuiltInTypes = 5 + def voluntaryAckType = UInt("b000") + def putAckType = UInt("b001") + def prefetchAckType = UInt("b011") + def getDataBeatType = UInt("b100") + def getDataBlockType = UInt("b101") + def typesWithData = Vec(getDataBlockType, getDataBeatType) + def typesWithMultibeatData= Vec(getDataBlockType) def apply( is_builtin_type: Bool, From f75126c39c9925fc2073a82e03c270372e6bb06d Mon Sep 17 00:00:00 2001 From: Albert Ou Date: Wed, 11 Mar 2015 23:24:58 -0700 Subject: [PATCH 298/374] Require self probes for all built-in Acquire types This ensures that puts by the RoCC accelerator properly invalidates its tile's L1 D$, with which it currently shares the same TileLink port. --- uncore/src/main/scala/tilelink.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 531d3f52..7ea79ad1 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -117,8 +117,9 @@ class Acquire extends ClientToManagerChannel def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && isBuiltInType() && Acquire.typesWithMultibeatData.contains(a_type) - //TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache: - def requiresSelfProbe(dummy: Int = 0) = isBuiltInType() && a_type === Acquire.getBlockType + //TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache + // and nbdcache invalidations after RoCC accelerator puts + def requiresSelfProbe(dummy: Int = 0) = isBuiltInType() def getBuiltInGrantType(dummy: Int = 0): UInt = { MuxLookup(this.a_type, Grant.putAckType, Array( From 8f8022379c11755367c0f35fce6a4775be91e06b Mon Sep 17 00:00:00 2001 From: Albert Ou Date: Wed, 11 Mar 2015 23:24:58 -0700 Subject: [PATCH 299/374] Fix AMO opcode extraction --- uncore/src/main/scala/tilelink.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 7ea79ad1..d353205e 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -96,7 +96,9 @@ class Acquire extends ClientToManagerChannel val addrByteOff = tlMemoryOperandSizeBits + opSizeOff val addrByteMSB = tlByteAddrBits + addrByteOff def allocate(dummy: Int = 0) = union(0) - def op_code(dummy: Int = 0) = Mux(hasData(), M_XWR, union(opSizeOff-1, opCodeOff)) + def op_code(dummy: Int = 0) = Mux(isBuiltInType() && + (a_type === Acquire.putType || a_type === Acquire.putBlockType), + M_XWR, union(opSizeOff-1, opCodeOff)) def op_size(dummy: Int = 0) = union(addrByteOff-1, opSizeOff) def addr_byte(dummy: Int = 0) = union(addrByteMSB-1, addrByteOff) def write_mask(dummy: Int = 0) = union(tlWriteMaskBits, 1) From 5e40c8ba77c7067fa686434db61cd9da4e3bb84f Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Thu, 12 Mar 2015 14:36:46 -0700 Subject: [PATCH 300/374] write back meta data when cache miss even when coherence meta data is clean --- uncore/src/main/scala/cache.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 87473587..9815855e 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -868,9 +868,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { is(s_inner_grant) { io.inner.grant.valid := Bool(true) when(ignt_data_done) { - val meta = pending_coh_on_ignt != xact_meta.coh - when(meta) { pending_coh := pending_coh_on_ignt } - state := Mux(meta, s_meta_write, + val coh_dirty = pending_coh_on_ignt != xact_meta.coh + when(coh_dirty) { pending_coh := pending_coh_on_ignt } + state := Mux(!is_hit || coh_dirty, s_meta_write, Mux(io.ignt().requiresAck(), s_inner_finish, s_idle)) } } From 2c31ed64261cafefb0273cb41dde5d002052dc73 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Thu, 12 Mar 2015 15:34:20 -0700 Subject: [PATCH 301/374] previous bug fix for meta data writeback wasn't quite right --- uncore/src/main/scala/cache.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 9815855e..7e601ba0 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -868,9 +868,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { is(s_inner_grant) { io.inner.grant.valid := Bool(true) when(ignt_data_done) { - val coh_dirty = pending_coh_on_ignt != xact_meta.coh - when(coh_dirty) { pending_coh := pending_coh_on_ignt } - state := Mux(!is_hit || coh_dirty, s_meta_write, + val meta_dirty = !is_hit || pending_coh_on_ignt != xact_meta.coh + when(meta_dirty) { pending_coh := pending_coh_on_ignt } + state := Mux(meta_dirty, s_meta_write, Mux(io.ignt().requiresAck(), s_inner_finish, s_idle)) } } From dcc84c4dd3e429235a4739ebc3edaf18f891f149 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 12 Mar 2015 09:33:56 -0700 Subject: [PATCH 302/374] arbiter probe ready bugfix --- uncore/src/main/scala/tilelink.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index d353205e..7da4cfab 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -572,7 +572,7 @@ abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule mngr: DecoupledIO[LogicalNetworkIO[M]]) { clts.map{ _.valid := mngr.valid } clts.map{ _.bits := mngr.bits } - mngr.ready := clts.map(_.ready).reduce(_||_) + mngr.ready := clts.map(_.ready).reduce(_&&_) } } From 8181262419de27535ce1fb396e82739533217fad Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 12 Mar 2015 16:22:14 -0700 Subject: [PATCH 303/374] clean up incoherent and probe flags --- uncore/src/main/scala/broadcast.scala | 30 ++++++++++++++------------- uncore/src/main/scala/cache.scala | 29 +++++++++++++------------- uncore/src/main/scala/uncore.scala | 2 +- 3 files changed, 32 insertions(+), 29 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index bd8a3f24..e28cbf1c 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -219,16 +219,15 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact Vec(Acquire.getType, Acquire.putType, Acquire.putAtomicType).contains(xact.a_type)), "Broadcast Hub does not support PutAtomics or subblock Gets/Puts") // TODO - val release_count = Reg(init=UInt(0, width = log2Up(nClients))) - val probe_flags = Reg(init=Bits(0, width = nClients)) - val curr_p_id = PriorityEncoder(probe_flags) - val probe_initial_flags = Bits(width = nClients) - probe_initial_flags := Bits(0) - // issue self-probes for uncached read xacts to facilitate I$ coherence + val release_count = Reg(init=UInt(0, width = log2Up(nCoherentClients+1))) + val pending_probes = Reg(init=Bits(0, width = nCoherentClients)) + val curr_p_id = PriorityEncoder(pending_probes) + val full_sharers = coh.full() val probe_self = io.inner.acquire.bits.payload.requiresSelfProbe() - val myflag = Mux(probe_self, Bits(0), - UIntToOH(io.inner.acquire.bits.header.src(log2Up(nClients)-1,0))) - probe_initial_flags := ~(io.incoherent.toBits | myflag) + val mask_self = Mux(probe_self, + full_sharers | UInt(UInt(1) << xact_src, width = nCoherentClients), + full_sharers & ~UInt(UInt(1) << xact_src, width = nCoherentClients)) + val mask_incoherent = mask_self & ~io.incoherent.toBits val collect_iacq_data = Reg(init=Bool(false)) val iacq_data_valid = Reg(init=Bits(0, width = innerDataBeats)) @@ -307,18 +306,21 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact data_buffer(UInt(0)) := io.iacq().data collect_iacq_data := io.iacq().hasMultibeatData() iacq_data_valid := io.iacq().hasData() << io.iacq().addr_beat - probe_flags := probe_initial_flags - release_count := PopCount(probe_initial_flags) - state := Mux(probe_initial_flags.orR, s_probe, + val needs_probes = mask_incoherent.orR + when(needs_probes) { + pending_probes := mask_incoherent + release_count := PopCount(mask_incoherent) + } + state := Mux(needs_probes, s_probe, Mux(pending_outer_write_, s_mem_write, Mux(pending_outer_read_, s_mem_read, s_make_grant))) } } is(s_probe) { // Generate probes - io.inner.probe.valid := probe_flags.orR + io.inner.probe.valid := pending_probes.orR when(io.inner.probe.ready) { - probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) + pending_probes := pending_probes & ~UIntToOH(curr_p_id) } // Handle releases, which may have data to be written back diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 7e601ba0..dfe5b10a 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -596,13 +596,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { dst = io.inner.grant.bits.header.dst), pending_coh.outer) - val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) - val pending_probes = Reg(init = Bits(0, width = nClients)) + val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) + val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) val curr_p_id = PriorityEncoder(pending_probes) val full_sharers = io.meta.resp.bits.meta.coh.inner.full() - val mask_self = Mux(xact.requiresSelfProbe(), - full_sharers | (UInt(1) << xact_src), - full_sharers & ~UInt(UInt(1) << xact_src, width = nClients)) + val probe_self = xact.requiresSelfProbe() + val mask_self = Mux(probe_self, + full_sharers | UInt(UInt(1) << xact_src, width = nCoherentClients), + full_sharers & ~UInt(UInt(1) << xact_src, width = nCoherentClients)) val mask_incoherent = mask_self & ~io.incoherent.toBits val collect_iacq_data = Reg(init=Bool(false)) @@ -770,8 +771,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val _needs_probes = _tag_match && _coh.inner.requiresProbes(xact) when(_is_hit) { pending_coh := pending_coh_on_hit } when(_needs_probes) { - pending_probes := mask_incoherent(nCoherentClients-1,0) - release_count := PopCount(mask_incoherent(nCoherentClients-1,0)) + pending_probes := mask_incoherent + release_count := PopCount(mask_incoherent) } state := Mux(_tag_match, Mux(_needs_probes, s_probe, Mux(_is_hit, s_data_read, s_outer_acquire)), // Probe, hit or upgrade @@ -920,9 +921,11 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val pending_finish = Reg{ io.outer.finish.bits.clone } val irel_had_data = Reg(init = Bool(false)) - val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) - val pending_probes = Reg(init = Bits(0, width = nClients)) + val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) + val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) val curr_p_id = PriorityEncoder(pending_probes) + val full_sharers = io.wb.req.bits.coh.inner.full() + val mask_incoherent = full_sharers & ~io.incoherent.toBits val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) val (orel_data_cnt, orel_data_done) = connectOutgoingDataBeatCounter(io.outer.release) @@ -985,12 +988,10 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { xact_way_en := io.wb.req.bits.way_en xact_id := io.wb.req.bits.id irel_had_data := Bool(false) - val coh = io.wb.req.bits.coh - val needs_probes = coh.inner.requiresProbesOnVoluntaryWriteback() + val needs_probes = io.wb.req.bits.coh.inner.requiresProbesOnVoluntaryWriteback() when(needs_probes) { - val mask_incoherent = coh.inner.full() & ~io.incoherent.toBits - pending_probes := mask_incoherent(nCoherentClients-1,0) - release_count := PopCount(mask_incoherent(nCoherentClients-1,0)) + pending_probes := mask_incoherent + release_count := PopCount(mask_incoherent) } state := Mux(needs_probes, s_probe, s_data_read) } diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 05319ee9..0d414f0d 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -73,7 +73,7 @@ trait HasCoherenceAgentWiringHelpers { trait HasInnerTLIO extends CoherenceAgentBundle { val inner = Bundle(new TileLinkIO)(innerTLParams).flip - val incoherent = Vec.fill(nClients){Bool()}.asInput + val incoherent = Vec.fill(nCoherentClients){Bool()}.asInput def iacq(dummy: Int = 0) = inner.acquire.bits.payload def iprb(dummy: Int = 0) = inner.probe.bits.payload def irel(dummy: Int = 0) = inner.release.bits.payload From 3a78ca210d27e194c47ff3216539de5ef6f248c5 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Thu, 12 Mar 2015 16:33:35 -0700 Subject: [PATCH 304/374] bugfix in uncached TL to TL convertors --- uncore/src/main/scala/tilelink.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 7da4cfab..1aac8ce4 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -443,7 +443,7 @@ class TileLinkIOWrapper extends TLModule { io.out.acquire <> io.in.acquire io.out.grant <> io.in.grant io.out.finish <> io.in.finish - io.out.probe.ready := Bool(false) + io.out.probe.ready := Bool(true) io.out.release.valid := Bool(false) } object TileLinkIOWrapper { @@ -484,7 +484,7 @@ class HeaderlessTileLinkIOWrapper extends TLModule { io.out.acquire <> io.in.acquire io.out.grant <> io.in.grant io.out.finish <> io.in.finish - io.out.probe.ready := Bool(false) + io.out.probe.ready := Bool(true) io.out.release.valid := Bool(false) } From 6e540825b214ac5c5a53a1f1e19fbaa213853145 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sat, 14 Mar 2015 02:15:24 -0700 Subject: [PATCH 305/374] Use entire 12-bit CSR address --- uncore/src/main/scala/htif.scala | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 644a8317..81d95453 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -34,7 +34,7 @@ class HostIO extends HTIFBundle class PCRReq extends Bundle { val rw = Bool() - val addr = Bits(width = 5) + val addr = Bits(width = 12) val data = Bits(width = 64) } @@ -206,7 +206,6 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { io.mem.finish.bits.payload.manager_xact_id := mem_gxid io.mem.finish.bits.header.dst := mem_gsrc - val pcr_reset = UInt(pcr_RESET)(pcr_addr.getWidth-1,0) val pcrReadData = Reg(Bits(width = io.cpu(0).pcr_rep.bits.getWidth)) for (i <- 0 until nCores) { val my_reset = Reg(init=Bool(true)) @@ -214,7 +213,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { val cpu = io.cpu(i) val me = pcr_coreid === UInt(i) - cpu.pcr_req.valid := state === state_pcr_req && me && pcr_addr != pcr_reset + cpu.pcr_req.valid := state === state_pcr_req && me && pcr_addr != UInt(pcr_RESET) cpu.pcr_req.bits.rw := cmd === cmd_writecr cpu.pcr_req.bits.addr := pcr_addr cpu.pcr_req.bits.data := pcr_wdata @@ -234,7 +233,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { when (cpu.pcr_req.valid && cpu.pcr_req.ready) { state := state_pcr_resp } - when (state === state_pcr_req && me && pcr_addr === pcr_reset) { + when (state === state_pcr_req && me && pcr_addr === UInt(pcr_RESET)) { when (cmd === cmd_writecr) { my_reset := pcr_wdata(0) } From c03976896e8908fc306e108d83e5dd759ae2a5db Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 15 Mar 2015 16:53:15 -0700 Subject: [PATCH 306/374] separate queues for resp tag and data --- uncore/src/main/scala/memserdes.scala | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index b9786b4c..0a2b9c39 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -33,9 +33,9 @@ class MemReqCmd extends HasMemAddr with HasMemTag { val rw = Bool() } -class MemResp extends HasMemData with HasMemTag - +class MemTag extends HasMemTag class MemData extends HasMemData +class MemResp extends HasMemData with HasMemTag class MemIO extends Bundle { val req_cmd = Decoupled(new MemReqCmd) @@ -543,9 +543,20 @@ class MemPipeIOMemIOConverter(numRequests: Int, refillCycles: Int) extends Modul io.mem.req_data <> io.cpu.req_data - val resp_dataq = Module((new HellaQueue(numEntries)) { new MemResp }) - resp_dataq.io.enq <> io.mem.resp - io.cpu.resp <> resp_dataq.io.deq + // Have separate queues to allow for different mem implementations + val resp_dataq = Module((new HellaQueue(numEntries)) { new MemData }) + resp_dataq.io.enq.valid := io.mem.resp.valid + resp_dataq.io.enq.bits.data := io.mem.resp.bits.data + + val resp_tagq = Module((new HellaQueue(numEntries)) { new MemTag }) + resp_tagq.io.enq.valid := io.mem.resp.valid + resp_tagq.io.enq.bits.tag := io.mem.resp.bits.tag + + io.cpu.resp.valid := resp_dataq.io.deq.valid && resp_tagq.io.deq.valid + io.cpu.resp.bits.data := resp_dataq.io.deq.bits.data + io.cpu.resp.bits.tag := resp_tagq.io.deq.bits.tag + resp_dataq.io.deq.ready := io.cpu.resp.ready + resp_tagq.io.deq.ready := io.cpu.resp.ready inc := resp_dataq.io.deq.fire() dec := io.mem.req_cmd.fire() && !io.mem.req_cmd.bits.rw From 23a6b007c1f6f357abfe2bad1209f8241bb31c9b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 15 Mar 2015 23:10:51 -0700 Subject: [PATCH 307/374] Fix BroadcastHub AcquiteTracker allocation bug and clean up tracker wiring --- uncore/src/main/scala/broadcast.scala | 100 +++++++++++++++----------- uncore/src/main/scala/cache.scala | 45 +++++++----- 2 files changed, 86 insertions(+), 59 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index e28cbf1c..c8014f45 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -40,53 +40,53 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent trackerList.map(_.io.incoherent := io.incoherent.toBits) // Queue to store impending Put data - val acquire = io.inner.acquire + val sdq = Vec.fill(sdqDepth){ Reg(io.iacq().data) } val sdq_val = Reg(init=Bits(0, sdqDepth)) val sdq_alloc_id = PriorityEncoder(~sdq_val) val sdq_rdy = !sdq_val.andR - val sdq_enq = acquire.fire() && acquire.bits.payload.hasData() - val sdq = Vec.fill(sdqDepth){ Reg(io.inner.acquire.bits.payload.data) } - when (sdq_enq) { sdq(sdq_alloc_id) := acquire.bits.payload.data } + val sdq_enq = io.inner.acquire.fire() && io.iacq().hasData() + when (sdq_enq) { sdq(sdq_alloc_id) := io.iacq().data } // Handle acquire transaction initiation - val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - val block_acquires = any_acquire_conflict val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) - for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.inner - alloc_arb.io.in(i).valid := t.acquire.ready - t.acquire.bits := acquire.bits - t.acquire.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits - t.acquire.valid := alloc_arb.io.in(i).ready + val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) + val acquireMatchList = trackerList.map(_.io.has_acquire_match) + val any_acquire_matches = acquireMatchList.reduce(_||_) + val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} + val match_idx = Vec(acquireMatchList).indexWhere{b: Bool => b} + val acquire_idx = Mux(any_acquire_matches, match_idx, alloc_idx) + trackerAcquireIOs.zip(alloc_arb.io.in).zipWithIndex.foreach { + case((tracker, arb), i) => + arb.valid := tracker.ready + tracker.bits := io.inner.acquire.bits + tracker.bits.payload.data := + DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits + tracker.valid := arb.ready && (acquire_idx === UInt(i)) } - acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && sdq_rdy && !block_acquires - alloc_arb.io.out.ready := acquire.valid && sdq_rdy && !block_acquires + val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) + io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && + sdq_rdy && !block_acquires + alloc_arb.io.out.ready := io.inner.acquire.valid && sdq_rdy && !block_acquires // Queue to store impending Voluntary Release data - val release = io.inner.release - val voluntary = release.bits.payload.isVoluntary() - val vwbdq_enq = release.fire() && voluntary && release.bits.payload.hasData() + val voluntary = io.irel().isVoluntary() + val vwbdq_enq = io.inner.release.fire() && voluntary && io.irel().hasData() val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, innerDataBeats) //TODO Zero width - val vwbdq = Vec.fill(innerDataBeats){ Reg(release.bits.payload.data) } //TODO Assumes nReleaseTransactors == 1 - when(vwbdq_enq) { vwbdq(rel_data_cnt) := release.bits.payload.data } + val vwbdq = Vec.fill(innerDataBeats){ Reg(io.irel().data) } //TODO Assumes nReleaseTransactors == 1 + when(vwbdq_enq) { vwbdq(rel_data_cnt) := io.irel().data } // Handle releases, which might be voluntary and might have data val release_idx = Vec(trackerList.map(_.io.has_release_match)).indexWhere{b: Bool => b} - for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.inner - t.release.bits := release.bits - t.release.bits.payload.data := (if (i < nReleaseTransactors) - DataQueueLocation(rel_data_cnt, inVolWBQueue) - else DataQueueLocation(UInt(0), inClientReleaseQueue)).toBits - t.release.valid := release.valid && (release_idx === UInt(i)) + val trackerReleaseIOs = trackerList.map(_.io.inner.release) + trackerReleaseIOs.zipWithIndex.foreach { + case(tracker, i) => + tracker.bits := io.inner.release.bits + tracker.bits.payload.data := DataQueueLocation(rel_data_cnt, + (if(i < nReleaseTransactors) inVolWBQueue + else inClientReleaseQueue)).toBits + tracker.valid := io.inner.release.valid && (release_idx === UInt(i)) } - release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) - - // Wire finished transaction acks - val ack = io.inner.finish - trackerList.map(_.io.inner.finish.valid := ack.valid) - trackerList.map(_.io.inner.finish.bits := ack.bits) - ack.ready := Bool(true) + io.inner.release.ready := Vec(trackerReleaseIOs.map(_.ready)).read(release_idx) // Wire probe requests and grant reply to clients, finish acks from clients // Note that we bypass the Grant data subbundles @@ -107,7 +107,7 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent val free_sdq = io.outer.acquire.fire() && io.outer.acquire.bits.payload.hasData() && outer_data_ptr.loc === inStoreQueue - io.outer.acquire.bits.payload.data := MuxLookup(outer_data_ptr.loc, release.bits.payload.data, Array( + io.outer.acquire.bits.payload.data := MuxLookup(outer_data_ptr.loc, io.irel().data, Array( inStoreQueue -> sdq(outer_data_ptr.idx), inVolWBQueue -> vwbdq(outer_data_ptr.idx))) io.outer <> outer_arb.io.out @@ -216,8 +216,9 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact val coh = ManagerMetadata.onReset assert(!(state != s_idle && xact.isBuiltInType() && - Vec(Acquire.getType, Acquire.putType, Acquire.putAtomicType).contains(xact.a_type)), - "Broadcast Hub does not support PutAtomics or subblock Gets/Puts") // TODO + Vec(Acquire.getType, Acquire.putType, Acquire.putAtomicType, + Acquire.prefetchType).contains(xact.a_type)), + "Broadcast Hub does not support PutAtomics, subblock Gets/Puts, or prefetches") // TODO val release_count = Reg(init=UInt(0, width = log2Up(nCoherentClients+1))) val pending_probes = Reg(init=Bits(0, width = nCoherentClients)) @@ -238,17 +239,18 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) val pending_ognt_ack = Reg(init=Bool(false)) val pending_outer_write = xact.hasData() - val pending_outer_read = io.ignt().hasData() val pending_outer_write_ = io.iacq().hasData() + val pending_outer_read = io.ignt().hasData() val pending_outer_read_ = coh.makeGrant(io.iacq(), UInt(trackerId)).hasData() io.has_acquire_conflict := xact.conflicts(io.iacq()) && - !collect_iacq_data && - (state != s_idle) - io.has_release_match := xact.conflicts(io.irel()) && - !io.irel().isVoluntary() && - (state != s_idle) - io.has_acquire_match := Bool(false) + (state != s_idle) && + !collect_iacq_data + io.has_acquire_match := xact.conflicts(io.iacq()) && + collect_iacq_data + io.has_release_match := xact.conflicts(io.irel()) && + !io.irel().isVoluntary() && + (state === s_probe) val outer_write_acq = Bundle(PutBlock( client_xact_id = UInt(trackerId), @@ -282,6 +284,18 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact io.inner.release.ready := Bool(false) io.inner.finish.ready := Bool(false) + assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && + io.inner.acquire.bits.header.src != xact_src), + "AcquireTracker accepted data beat from different network source than initial request.") + + assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && + io.iacq().client_xact_id != xact.client_xact_id), + "AcquireTracker accepted data beat from different client transaction than initial request.") + + assert(!(state === s_idle && io.inner.acquire.fire() && + io.iacq().addr_beat != UInt(0)), + "AcquireTracker initialized with a tail data beat.") + when(collect_iacq_data) { io.inner.acquire.ready := Bool(true) when(io.inner.acquire.valid) { diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index dfe5b10a..ef3cb3d7 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -356,33 +356,34 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule (trackerList.map(_.io.incoherent) :+ wb.io.incoherent).map( _ := io.incoherent.toBits) // Handle acquire transaction initiation - val acquire = io.inner.acquire val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) - val acquireList = trackerList.map(_.io.inner.acquire) + val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) val acquireMatchList = trackerList.map(_.io.has_acquire_match) val any_acquire_matches = acquireMatchList.reduce(_||_) val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} val match_idx = Vec(acquireMatchList).indexWhere{b: Bool => b} val acquire_idx = Mux(any_acquire_matches, match_idx, alloc_idx) - acquireList.zip(alloc_arb.io.in).zipWithIndex.map { case((acq, arb), i) => - arb.valid := acq.ready - acq.bits := acquire.bits - acq.valid := arb.ready && (acquire_idx === UInt(i)) + trackerAcquireIOs.zip(alloc_arb.io.in).zipWithIndex.foreach { + case((tracker, arb), i) => + arb.valid := tracker.ready + tracker.bits := io.inner.acquire.bits + tracker.valid := arb.ready && (acquire_idx === UInt(i)) } val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - acquire.ready := acquireList.map(_.ready).reduce(_||_) && !block_acquires - alloc_arb.io.out.ready := acquire.valid && !block_acquires + io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && + !block_acquires + alloc_arb.io.out.ready := io.inner.acquire.valid && !block_acquires // Wire releases from clients - val release = io.inner.release val release_idx = Vec(trackerList.map(_.io.has_release_match) :+ wb.io.has_release_match).indexWhere{b: Bool => b} - val releaseList = trackerList.map(_.io.inner.release) :+ wb.io.inner.release - releaseList.zipWithIndex.map { case(r, i) => - r.bits := release.bits - r.valid := release.valid && (release_idx === UInt(i)) + val trackerReleaseIOs = trackerList.map(_.io.inner.release) :+ wb.io.inner.release + trackerReleaseIOs.zipWithIndex.foreach { + case(tracker, i) => + tracker.bits := io.inner.release.bits + tracker.valid := io.inner.release.valid && (release_idx === UInt(i)) } - release.ready := Vec(releaseList.map(_.ready)).read(release_idx) + io.inner.release.ready := Vec(trackerReleaseIOs.map(_.ready)).read(release_idx) // Wire probe requests and grant reply to clients, finish acks from clients doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe) :+ wb.io.inner.probe) @@ -658,8 +659,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { !collect_iacq_data io.has_acquire_match := xact.conflicts(io.iacq()) && collect_iacq_data - io.has_release_match := !io.irel().isVoluntary() && - (xact.addr_block === io.irel().addr_block) && + io.has_release_match := xact.conflicts(io.irel()) && + !io.irel().isVoluntary() && (state === s_probe) // If we're allocating in this cache, we can use the current metadata @@ -729,6 +730,18 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.wb.req.bits.way_en := xact_way_en io.wb.req.bits.id := UInt(trackerId) + assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && + io.inner.acquire.bits.header.src != xact_src), + "AcquireTracker accepted data beat from different network source than initial request.") + + assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && + io.iacq().client_xact_id != xact.client_xact_id), + "AcquireTracker accepted data beat from different client transaction than initial request.") + + assert(!(state === s_idle && io.inner.acquire.fire() && + io.iacq().addr_beat != UInt(0)), + "AcquireTracker initialized with a tail data beat.") + when(collect_iacq_data) { io.inner.acquire.ready := Bool(true) when(io.inner.acquire.valid) { From f6d1a2fb76cfe73ff87149264e9ac2f1237a82bb Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 16 Mar 2015 00:09:38 -0700 Subject: [PATCH 308/374] No more self-probes required --- uncore/src/main/scala/tilelink.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 1aac8ce4..7bd218ad 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -119,9 +119,7 @@ class Acquire extends ClientToManagerChannel def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && isBuiltInType() && Acquire.typesWithMultibeatData.contains(a_type) - //TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache - // and nbdcache invalidations after RoCC accelerator puts - def requiresSelfProbe(dummy: Int = 0) = isBuiltInType() + def requiresSelfProbe(dummy: Int = 0) = Bool(false) def getBuiltInGrantType(dummy: Int = 0): UInt = { MuxLookup(this.a_type, Grant.putAckType, Array( From b72230a9f04f18d9358ca427b69445ff090e6d95 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 16 Mar 2015 00:09:55 -0700 Subject: [PATCH 309/374] PutBlock bugfix --- uncore/src/main/scala/cache.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index ef3cb3d7..8c65a20f 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -636,7 +636,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val offset = xact.addr_byte()(byteAddrBits-1, log2Up(amoAluOperandBits/8)) amoalu.io.lhs := old_data >> offset*amoOpSz amoalu.io.rhs := new_data >> offset*amoOpSz - val valid_beat = xact.addr_beat === beat + val valid_beat = xact.is(Acquire.putBlockType) || xact.addr_beat === beat val wmask = Fill(dataBits, valid_beat) & Mux(xact.is(Acquire.putAtomicType), FillInterleaved(amoAluOperandBits, UIntToOH(offset)), From f35a6a574f3386d282f9fcd9ef1fd692fe9e9c1c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 16 Mar 2015 13:25:01 -0700 Subject: [PATCH 310/374] Add a queue on released data coming in to L2 --- uncore/src/main/scala/cache.scala | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 8c65a20f..bc12cfeb 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -322,14 +322,22 @@ class L2HellaCacheBank(bankId: Int) extends HierarchicalCoherenceAgent require(isPow2(nWays)) val tshrfile = Module(new TSHRFile(bankId)) - val meta = Module(new L2MetadataArray) - val data = Module(new L2DataArray(1)) - tshrfile.io.inner <> io.inner - tshrfile.io.meta <> meta.io - tshrfile.io.data <> data.io + //TODO: Expose queue depths and data array pipeline cycles as parameters? + tshrfile.io.inner.acquire <> io.inner.acquire + tshrfile.io.inner.probe <> io.inner.probe + tshrfile.io.inner.release <> Queue(io.inner.release) + tshrfile.io.inner.grant <> io.inner.grant + tshrfile.io.inner.finish <> io.inner.finish + io.outer <> tshrfile.io.outer io.incoherent <> tshrfile.io.incoherent + + val meta = Module(new L2MetadataArray) + tshrfile.io.meta <> meta.io + + val data = Module(new L2DataArray(1)) + tshrfile.io.data <> data.io } class TSHRFileIO extends HierarchicalTLIO { From cf1df2d72a3b3cd687eed7f7ed8c3a17ac62758e Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 16 Mar 2015 13:27:05 -0700 Subject: [PATCH 311/374] further amo cleanups --- uncore/src/main/scala/cache.scala | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index bc12cfeb..e1c8bce4 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -572,11 +572,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val state = Reg(init=s_idle) val xact_src = Reg(io.inner.acquire.bits.header.src.clone) - val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId) })) val data_buffer = Vec.fill(innerDataBeats) { Reg(io.iacq().data.clone) } - val amo_result = Reg(io.iacq().data.clone) + val amo_result = xact.data val xact_tag_match = Reg{ Bool() } val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } @@ -638,13 +638,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { def mergeData[T <: HasTileLinkData] (byteAddrBits: Int, dataBits: Int) (buffer: Vec[UInt], beat: UInt, incoming: UInt) { - val old_data = incoming - val new_data = buffer(beat) + val old_data = incoming // Refilled, written back, or de-cached data + val new_data = buffer(beat) // Newly Put data is in the buffer val amoOpSz = UInt(amoAluOperandBits) val offset = xact.addr_byte()(byteAddrBits-1, log2Up(amoAluOperandBits/8)) amoalu.io.lhs := old_data >> offset*amoOpSz amoalu.io.rhs := new_data >> offset*amoOpSz - val valid_beat = xact.is(Acquire.putBlockType) || xact.addr_beat === beat + val valid_beat = (xact.is(Acquire.putBlockType) || xact.addr_beat === beat) && + xact.isBuiltInType() // Only custom a_types have data for now val wmask = Fill(dataBits, valid_beat) & Mux(xact.is(Acquire.putAtomicType), FillInterleaved(amoAluOperandBits, UIntToOH(offset)), @@ -746,6 +747,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.iacq().client_xact_id != xact.client_xact_id), "AcquireTracker accepted data beat from different client transaction than initial request.") + //TODO: Assumes in-order network assert(!(state === s_idle && io.inner.acquire.fire() && io.iacq().addr_beat != UInt(0)), "AcquireTracker initialized with a tail data beat.") @@ -765,6 +767,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.inner.acquire.valid) { xact_src := io.inner.acquire.bits.header.src xact := io.iacq() + xact.data := UInt(0) data_buffer(io.iacq().addr_beat) := io.iacq().data collect_iacq_data := io.iacq().hasMultibeatData() iacq_data_valid := io.iacq().hasData() << io.iacq().addr_beat From 145e15701e8e2dc6b34c252487201c0801b0d8cb Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Mon, 16 Mar 2015 18:47:16 -0700 Subject: [PATCH 312/374] bugfix where an in-progress acquire can be blocked by another acquire tracker being free'd up in between --- uncore/src/main/scala/cache.scala | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index e1c8bce4..da000c5e 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -364,23 +364,28 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule (trackerList.map(_.io.incoherent) :+ wb.io.incoherent).map( _ := io.incoherent.toBits) // Handle acquire transaction initiation + val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) + val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) - val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) + alloc_arb.io.out.ready := Bool(true) + trackerAcquireIOs.zip(alloc_arb.io.in).foreach { + case(tracker, arb) => + arb.valid := tracker.ready + } + val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} + val acquireMatchList = trackerList.map(_.io.has_acquire_match) val any_acquire_matches = acquireMatchList.reduce(_||_) - val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} val match_idx = Vec(acquireMatchList).indexWhere{b: Bool => b} + val acquire_idx = Mux(any_acquire_matches, match_idx, alloc_idx) - trackerAcquireIOs.zip(alloc_arb.io.in).zipWithIndex.foreach { - case((tracker, arb), i) => - arb.valid := tracker.ready - tracker.bits := io.inner.acquire.bits - tracker.valid := arb.ready && (acquire_idx === UInt(i)) - } val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && - !block_acquires - alloc_arb.io.out.ready := io.inner.acquire.valid && !block_acquires + io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && !block_acquires + trackerAcquireIOs.zipWithIndex.foreach { + case(tracker, i) => + tracker.bits := io.inner.acquire.bits + tracker.valid := io.inner.acquire.valid && !block_acquires && (acquire_idx === UInt(i)) + } // Wire releases from clients val release_idx = Vec(trackerList.map(_.io.has_release_match) :+ From 36fc67dc7c1011afa763350c819657a911fdd74b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 16 Mar 2015 15:06:46 -0700 Subject: [PATCH 313/374] cleanup mergeData buffer --- uncore/src/main/scala/cache.scala | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index da000c5e..b6d22d5e 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -642,23 +642,23 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { def mergeData[T <: HasTileLinkData] (byteAddrBits: Int, dataBits: Int) - (buffer: Vec[UInt], beat: UInt, incoming: UInt) { + (beat: UInt, incoming: UInt) { val old_data = incoming // Refilled, written back, or de-cached data - val new_data = buffer(beat) // Newly Put data is in the buffer + val new_data = data_buffer(beat) // Newly Put data is in the buffer val amoOpSz = UInt(amoAluOperandBits) - val offset = xact.addr_byte()(byteAddrBits-1, log2Up(amoAluOperandBits/8)) - amoalu.io.lhs := old_data >> offset*amoOpSz - amoalu.io.rhs := new_data >> offset*amoOpSz + val amoOffset = xact.addr_byte()(byteAddrBits-1, log2Up(amoAluOperandBits/8)) + amoalu.io.lhs := old_data >> amoOffset*amoOpSz + amoalu.io.rhs := new_data >> amoOffset*amoOpSz val valid_beat = (xact.is(Acquire.putBlockType) || xact.addr_beat === beat) && xact.isBuiltInType() // Only custom a_types have data for now val wmask = Fill(dataBits, valid_beat) & Mux(xact.is(Acquire.putAtomicType), - FillInterleaved(amoAluOperandBits, UIntToOH(offset)), + FillInterleaved(amoAluOperandBits, UIntToOH(amoOffset)), Mux(xact.is(Acquire.putBlockType) || xact.is(Acquire.putType), - FillInterleaved(8, xact.write_mask()), + FillInterleaved(8, write_mask_buffer(beat)), UInt(0, width = dataBits))) - buffer(beat) := ~wmask & old_data | wmask & - Mux(xact.is(Acquire.putAtomicType), amoalu.io.out << offset*amoOpSz, new_data) + data_buffer(beat) := ~wmask & old_data | wmask & + Mux(xact.is(Acquire.putAtomicType), amoalu.io.out << amoOffset*amoOpSz, new_data) when(xact.is(Acquire.putAtomicType) && valid_beat) { amo_result := old_data } } val mergeDataInternal = mergeData(log2Up(rowBits/8), rowBits) _ @@ -831,7 +831,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.irel().hasData()) { irel_had_data := Bool(true) pending_coh.outer := pending_ocoh_on_irel - mergeDataInner(data_buffer, io.irel().addr_beat, io.irel().data) + mergeDataInner(io.irel().addr_beat, io.irel().data) } // We don't decrement release_count until we've received all the data beats. when(!io.irel().hasMultibeatData() || irel_data_done) { @@ -852,7 +852,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.grant.ready := Bool(true) when(io.outer.grant.valid) { when(io.ognt().hasData()) { - mergeDataOuter(data_buffer, io.ognt().addr_beat, io.ognt().data) + mergeDataOuter(io.ognt().addr_beat, io.ognt().data) ognt_had_data := Bool(true) } when(ognt_data_done) { @@ -879,13 +879,13 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { is(s_data_read) { io.data.read.valid := !collect_iacq_data || iacq_data_valid(read_data_cnt) when(io.data.resp.valid) { - mergeDataInternal(data_buffer, io.data.resp.bits.addr_beat, io.data.resp.bits.data) + mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) } when(read_data_done) { state := s_data_resp } } is(s_data_resp) { when(io.data.resp.valid) { - mergeDataInternal(data_buffer, io.data.resp.bits.addr_beat, io.data.resp.bits.data) + mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) } when(resp_data_done) { state := Mux(xact.hasData(), s_data_write, s_inner_grant) From f6fe037e303d708101c0d1d7b5716a9efb3919ab Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 16 Mar 2015 23:41:56 -0700 Subject: [PATCH 314/374] first cut of merging puts/gets --- uncore/src/main/scala/cache.scala | 260 ++++++++++++++++----------- uncore/src/main/scala/tilelink.scala | 24 ++- 2 files changed, 170 insertions(+), 114 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index b6d22d5e..a1defb3b 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -171,6 +171,9 @@ abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgen val amoAluOperandBits = params(AmoAluOperandBits) require(amoAluOperandBits <= innerDataBits) require(rowBits == innerDataBits) // TODO: relax this by improving s_data_* states + val nSecondaryMisses = 4 + val enableGetMerging = false + val enablePutMerging = true } abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters @@ -316,6 +319,10 @@ class L2DataArray(delay: Int) extends L2HellaCacheModule { io.write.ready := Bool(true) } +class L2SecondaryMissInfo extends TLBundle + with HasTileLinkBeatId + with HasClientTransactionId + class L2HellaCacheBank(bankId: Int) extends HierarchicalCoherenceAgent with L2HellaCacheParameters { require(isPow2(nSets)) @@ -570,6 +577,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack } } + class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val io = new L2XactTrackerIO @@ -578,15 +586,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_src = Reg(io.inner.acquire.bits.header.src.clone) val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId) })) - val data_buffer = Vec.fill(innerDataBeats) { - Reg(io.iacq().data.clone) - } - val amo_result = xact.data + val data_buffer = Vec.fill(innerDataBeats){ Reg(UInt(width = innerDataBits)) } + val wmask_buffer = Vec.fill(innerDataBeats){ Reg(Bits(width = innerDataBits/8)) } val xact_tag_match = Reg{ Bool() } val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } val pending_coh = Reg{ xact_meta.coh.clone } val pending_finish = Reg{ io.outer.finish.bits.clone } + val ignt_q = Module(new Queue(new L2SecondaryMissInfo, nSecondaryMisses))(innerTLParams) val is_hit = xact_tag_match && xact_meta.coh.outer.isHit(xact.op_code()) val do_allocate = xact.allocate() @@ -594,6 +601,29 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { (xact_meta.coh.outer.requiresVoluntaryWriteback() || xact_meta.coh.inner.requiresProbesOnVoluntaryWriteback()) + val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) + val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) + val curr_probe_dst = PriorityEncoder(pending_probes) + val full_sharers = io.meta.resp.bits.meta.coh.inner.full() + val probe_self = xact.requiresSelfProbe() + val mask_self = Mux(probe_self, + full_sharers | UInt(UInt(1) << xact_src, width = nCoherentClients), + full_sharers & ~UInt(UInt(1) << xact_src, width = nCoherentClients)) + val mask_incoherent = mask_self & ~io.incoherent.toBits + + val irel_had_data = Reg(init = Bool(false)) + val ognt_had_data = Reg(init = Bool(false)) + val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) + val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire, xact.addr_beat) + val (ignt_data_idx, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) + + val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) + val pending_writes = Reg(init=Bits(0, width = innerDataBeats)) + val pending_resps = Reg(init=Bits(0, width = innerDataBeats)) + val curr_read_beat = PriorityEncoder(pending_reads) + val curr_write_beat = PriorityEncoder(pending_writes) + val pending_coh_on_hit = HierarchicalMetadata( io.meta.resp.bits.meta.coh.inner, io.meta.resp.bits.meta.coh.outer.onHit(xact.op_code())) @@ -610,29 +640,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { dst = io.inner.grant.bits.header.dst), pending_coh.outer) - val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) - val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) - val curr_p_id = PriorityEncoder(pending_probes) - val full_sharers = io.meta.resp.bits.meta.coh.inner.full() - val probe_self = xact.requiresSelfProbe() - val mask_self = Mux(probe_self, - full_sharers | UInt(UInt(1) << xact_src, width = nCoherentClients), - full_sharers & ~UInt(UInt(1) << xact_src, width = nCoherentClients)) - val mask_incoherent = mask_self & ~io.incoherent.toBits - - val collect_iacq_data = Reg(init=Bool(false)) - val iacq_data_valid = Reg(init=Bits(0, width = innerDataBeats)) - val irel_had_data = Reg(init = Bool(false)) - val ognt_had_data = Reg(init = Bool(false)) - val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) - val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) - val (ignt_data_cnt, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, xact.addr_beat) - val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire, xact.addr_beat) - val (read_data_cnt, read_data_done) = connectInternalDataBeatCounter(io.data.read, xact.addr_beat, !xact.isSubBlockType()) - val (write_data_cnt, write_data_done) = connectInternalDataBeatCounter(io.data.write, xact.addr_beat, !xact.isSubBlockType() || ognt_had_data || irel_had_data) - val resp_data_done = connectInternalDataBeatCounter(io.data.resp, !xact.isSubBlockType()) - + val amo_result = xact.data val amoalu = Module(new AMOALU) amoalu.io.addr := xact.addr() amoalu.io.cmd := xact.op_code() @@ -640,42 +648,62 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { amoalu.io.lhs := io.data.resp.bits.data //default amoalu.io.rhs := data_buffer.head // default - def mergeData[T <: HasTileLinkData] - (byteAddrBits: Int, dataBits: Int) - (beat: UInt, incoming: UInt) { + def mergeDataPut(beat: UInt, wmask: UInt, put_data: UInt) { + data_buffer(beat) := ~wmask & data_buffer(beat) | wmask & put_data + } + + def mergeData(dataBits: Int)(beat: UInt, incoming: UInt) { val old_data = incoming // Refilled, written back, or de-cached data - val new_data = data_buffer(beat) // Newly Put data is in the buffer - val amoOpSz = UInt(amoAluOperandBits) - val amoOffset = xact.addr_byte()(byteAddrBits-1, log2Up(amoAluOperandBits/8)) - amoalu.io.lhs := old_data >> amoOffset*amoOpSz - amoalu.io.rhs := new_data >> amoOffset*amoOpSz - val valid_beat = (xact.is(Acquire.putBlockType) || xact.addr_beat === beat) && - xact.isBuiltInType() // Only custom a_types have data for now - val wmask = Fill(dataBits, valid_beat) & - Mux(xact.is(Acquire.putAtomicType), - FillInterleaved(amoAluOperandBits, UIntToOH(amoOffset)), - Mux(xact.is(Acquire.putBlockType) || xact.is(Acquire.putType), - FillInterleaved(8, write_mask_buffer(beat)), - UInt(0, width = dataBits))) - data_buffer(beat) := ~wmask & old_data | wmask & - Mux(xact.is(Acquire.putAtomicType), amoalu.io.out << amoOffset*amoOpSz, new_data) + val new_data = data_buffer(beat) // Newly Put data is already in the buffer + amoalu.io.lhs := old_data >> xact.amo_shift_bits() + amoalu.io.rhs := new_data >> xact.amo_shift_bits() + val valid_beat = (xact.is(Acquire.putBlockType) || xact.addr_beat === beat) + val wmask = Fill(dataBits, valid_beat) & wmask_buffer(beat) + data_buffer(beat) := ~wmask & old_data | + wmask & Mux(xact.is(Acquire.putAtomicType), + amoalu.io.out << xact.amo_shift_bits(), + new_data) when(xact.is(Acquire.putAtomicType) && valid_beat) { amo_result := old_data } } - val mergeDataInternal = mergeData(log2Up(rowBits/8), rowBits) _ - val mergeDataInner = mergeData(innerByteAddrBits, innerDataBits) _ - val mergeDataOuter = mergeData(outerByteAddrBits, outerDataBits) _ + val mergeDataInternal = mergeData(rowBits) _ + val mergeDataInner = mergeData(innerDataBits) _ + val mergeDataOuter = mergeData(outerDataBits) _ + + val can_merge_iacq_get = Bool(enableGetMerging) && + (xact.isBuiltInType(Acquire.getType) && + io.iacq().isBuiltInType(Acquire.getType)) && + (xact_src === io.inner.acquire.bits.header.src) && + xact.conflicts(io.iacq()) && + Vec(s_meta_read, s_meta_resp, s_wb_req, s_wb_resp, + s_probe, s_outer_acquire, s_outer_grant, + s_outer_finish).contains(state) && + do_allocate && + ignt_q.io.enq.ready + //TODO: mix Puts and PutBlocks + val can_merge_iacq_put = ((Bool(enablePutMerging) && + (xact.isBuiltInType(Acquire.putType) && + io.iacq().isBuiltInType(Acquire.putType))) || + (xact.isBuiltInType(Acquire.putBlockType) && + io.iacq().isBuiltInType(Acquire.putBlockType))) && + (xact_src === io.inner.acquire.bits.header.src) && + (xact.client_xact_id === io.iacq().client_xact_id) && + xact.conflicts(io.iacq()) && + Vec(s_meta_read, s_meta_resp, s_wb_req, s_wb_resp, + s_probe, s_outer_acquire, s_outer_grant, + s_outer_finish, s_data_read, + s_data_resp).contains(state) && + do_allocate && + ignt_q.io.enq.ready - //TODO: Allow hit under miss for stores val in_same_set = xact.addr_block(idxMSB,idxLSB) === io.iacq().addr_block(idxMSB,idxLSB) - io.has_acquire_conflict := (xact.conflicts(io.iacq()) || in_same_set) && - (state != s_idle) && - !collect_iacq_data - io.has_acquire_match := xact.conflicts(io.iacq()) && - collect_iacq_data io.has_release_match := xact.conflicts(io.irel()) && !io.irel().isVoluntary() && (state === s_probe) + io.has_acquire_match := can_merge_iacq_put || can_merge_iacq_get + io.has_acquire_conflict := (xact.conflicts(io.iacq()) || in_same_set) && + (state != s_idle) && + !io.has_acquire_match // If we're allocating in this cache, we can use the current metadata // to make an appropriate custom Acquire, otherwise we copy over the @@ -697,21 +725,25 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_p_id + io.inner.probe.bits.header.dst := curr_probe_dst io.inner.probe.bits.payload := pending_coh.inner.makeProbe(xact) - io.inner.grant.valid := Bool(false) + io.inner.grant.valid := state === s_inner_grant && ignt_q.io.deq.valid io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := xact_src io.inner.grant.bits.payload := pending_coh.inner.makeGrant( acq = xact, manager_xact_id = UInt(trackerId), - addr_beat = ignt_data_cnt, + addr_beat = ignt_data_idx, data = Mux(xact.is(Acquire.putAtomicType), amo_result, - data_buffer(ignt_data_cnt))) + data_buffer(ignt_data_idx))) + io.ignt().client_xact_id := ignt_q.io.deq.bits.client_xact_id + ignt_q.io.deq.ready := ignt_data_done - io.inner.acquire.ready := Bool(false) + io.inner.acquire.ready := state === s_idle || + can_merge_iacq_put || + can_merge_iacq_get io.inner.release.ready := Bool(false) io.inner.finish.ready := Bool(false) @@ -719,14 +751,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.data.read.bits.id := UInt(trackerId) io.data.read.bits.way_en := xact_way_en io.data.read.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) - io.data.read.bits.addr_beat := read_data_cnt + io.data.read.bits.addr_beat := curr_read_beat io.data.write.valid := Bool(false) io.data.write.bits.id := UInt(trackerId) io.data.write.bits.way_en := xact_way_en io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) - io.data.write.bits.addr_beat := write_data_cnt + io.data.write.bits.addr_beat := curr_write_beat io.data.write.bits.wmask := SInt(-1) - io.data.write.bits.data := data_buffer(write_data_cnt) + io.data.write.bits.data := data_buffer(curr_write_beat) io.meta.read.valid := Bool(false) io.meta.read.bits.id := UInt(trackerId) io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) @@ -744,38 +776,16 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.wb.req.bits.way_en := xact_way_en io.wb.req.bits.id := UInt(trackerId) - assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && - io.inner.acquire.bits.header.src != xact_src), - "AcquireTracker accepted data beat from different network source than initial request.") - - assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && - io.iacq().client_xact_id != xact.client_xact_id), - "AcquireTracker accepted data beat from different client transaction than initial request.") - - //TODO: Assumes in-order network - assert(!(state === s_idle && io.inner.acquire.fire() && - io.iacq().addr_beat != UInt(0)), - "AcquireTracker initialized with a tail data beat.") - - when(collect_iacq_data) { - io.inner.acquire.ready := Bool(true) - when(io.inner.acquire.valid) { - data_buffer(io.iacq().addr_beat) := io.iacq().data - iacq_data_valid(io.iacq().addr_beat) := Bool(true) - } - when(iacq_data_done) { collect_iacq_data := Bool(false) } - } - switch (state) { is(s_idle) { - io.inner.acquire.ready := Bool(true) when(io.inner.acquire.valid) { xact_src := io.inner.acquire.bits.header.src xact := io.iacq() xact.data := UInt(0) - data_buffer(io.iacq().addr_beat) := io.iacq().data - collect_iacq_data := io.iacq().hasMultibeatData() - iacq_data_valid := io.iacq().hasData() << io.iacq().addr_beat + wmask_buffer.foreach { w => w := UInt(0) } + pending_reads := Mux(io.iacq().isSubBlockType(), SInt(0), SInt(-1)).toUInt + pending_writes := UInt(0) + pending_resps := UInt(0) irel_had_data := Bool(false) ognt_had_data := Bool(false) state := s_meta_read @@ -819,7 +829,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { // Send probes io.inner.probe.valid := pending_probes != UInt(0) when(io.inner.probe.ready) { - pending_probes := pending_probes & ~UIntToOH(curr_p_id) + pending_probes := pending_probes & ~UIntToOH(curr_probe_dst) } // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) @@ -829,9 +839,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { //TODO: make sure cacq data is actually present before accpeting // release data to merge! when(io.irel().hasData()) { - irel_had_data := Bool(true) pending_coh.outer := pending_ocoh_on_irel mergeDataInner(io.irel().addr_beat, io.irel().data) + pending_writes := pending_writes | UIntToOH(io.irel().addr_beat) } // We don't decrement release_count until we've received all the data beats. when(!io.irel().hasMultibeatData() || irel_data_done) { @@ -839,11 +849,13 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } } when(release_count === UInt(0)) { - state := Mux(is_hit, Mux(irel_had_data, s_data_write, s_data_read), s_outer_acquire) + state := Mux(is_hit, + Mux(pending_writes.orR, s_data_write, s_data_read), + s_outer_acquire) } } is(s_outer_acquire) { - io.outer.acquire.valid := !iacq_data_done // collect all data before refilling + io.outer.acquire.valid := Bool(true) when(oacq_data_done) { state := s_outer_grant } @@ -853,7 +865,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.outer.grant.valid) { when(io.ognt().hasData()) { mergeDataOuter(io.ognt().addr_beat, io.ognt().data) - ognt_had_data := Bool(true) + pending_writes := pending_writes | UIntToOH(io.ognt().addr_beat) } when(ognt_data_done) { pending_coh := pending_coh_on_ognt @@ -864,7 +876,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { state := s_outer_finish }.otherwise { state := Mux(!do_allocate, s_inner_grant, - Mux(io.ognt().hasData(), s_data_write, s_data_read)) + Mux(pending_writes.orR, s_data_write, s_data_read)) } } } @@ -873,31 +885,39 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.finish.valid := Bool(true) when(io.outer.finish.ready) { state := Mux(!do_allocate, s_inner_grant, - Mux(ognt_had_data, s_data_write, s_data_read)) + Mux(pending_writes.orR, s_data_write, s_data_read)) } } is(s_data_read) { - io.data.read.valid := !collect_iacq_data || iacq_data_valid(read_data_cnt) + io.data.read.valid := pending_reads.orR + when(io.data.read.ready) { + pending_resps := pending_resps | UIntToOH(curr_read_beat) + pending_reads := pending_reads & ~UIntToOH(curr_read_beat) + when(PopCount(pending_reads) <= UInt(1)) { state := s_data_resp } + } when(io.data.resp.valid) { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) + pending_resps := pending_resps & ~UIntToOH(io.data.resp.bits.addr_beat) } - when(read_data_done) { state := s_data_resp } } is(s_data_resp) { when(io.data.resp.valid) { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) - } - when(resp_data_done) { - state := Mux(xact.hasData(), s_data_write, s_inner_grant) + pending_resps := pending_resps & ~UIntToOH(io.data.resp.bits.addr_beat) + when(PopCount(pending_resps) <= UInt(1)) { + state := Mux(pending_writes.orR, s_data_write, s_inner_grant) + } } } is(s_data_write) { - io.data.write.valid := Bool(true) - when(write_data_done) { state := s_inner_grant } + io.data.write.valid := pending_writes.orR //TODO make sure all acquire data is present + when(io.data.write.ready) { + pending_writes := pending_writes & ~UIntToOH(curr_write_beat) + when(PopCount(pending_writes) <= UInt(1)) { state := s_inner_grant } + } } is(s_inner_grant) { - io.inner.grant.valid := Bool(true) - when(ignt_data_done) { + when(ignt_data_done && ignt_q.io.count === UInt(1)) { val meta_dirty = !is_hit || pending_coh_on_ignt != xact_meta.coh when(meta_dirty) { pending_coh := pending_coh_on_ignt } state := Mux(meta_dirty, s_meta_write, @@ -915,6 +935,32 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.inner.finish.valid) { state := s_idle } } } + + ignt_q.io.enq.valid := io.inner.acquire.fire() && + (state === s_idle || !xact.hasMultibeatData()) + ignt_q.io.enq.bits.client_xact_id := io.iacq().client_xact_id + ignt_q.io.enq.bits.addr_beat := io.iacq().addr_beat + + // Handle Get and Put merging + when(io.inner.acquire.fire()) { + val beat = io.iacq().addr_beat + when(io.iacq().hasData()) { + mergeDataPut(beat, io.iacq().wmask(), io.iacq().data) + wmask_buffer(beat) := io.iacq().wmask() | wmask_buffer(beat) + //iacq_data_valid(beat) := Bool(true) + pending_writes(beat) := Bool(true) + } + pending_reads(beat) := Bool(true) + } + + assert(!(state != s_idle && io.inner.acquire.fire() && + io.inner.acquire.bits.header.src != xact_src), + "AcquireTracker accepted data beat from different network source than initial request.") + + //TODO: Assumes in-order network + assert(!(state === s_idle && io.inner.acquire.fire() && + io.iacq().addr_beat != UInt(0)), + "AcquireTracker initialized with a tail data beat.") } class L2WritebackReq extends L2HellaCacheBundle @@ -952,7 +998,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val irel_had_data = Reg(init = Bool(false)) val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) - val curr_p_id = PriorityEncoder(pending_probes) + val curr_probe_dst = PriorityEncoder(pending_probes) val full_sharers = io.wb.req.bits.coh.inner.full() val mask_incoherent = full_sharers & ~io.incoherent.toBits @@ -988,7 +1034,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_p_id + io.inner.probe.bits.header.dst := curr_probe_dst io.inner.probe.bits.payload := xact_coh.inner.makeProbeForVoluntaryWriteback(xact_addr_block) @@ -1029,7 +1075,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { // Send probes io.inner.probe.valid := pending_probes != UInt(0) when(io.inner.probe.ready) { - pending_probes := pending_probes & ~UIntToOH(curr_p_id) + pending_probes := pending_probes & ~UIntToOH(curr_probe_dst) } // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 7bd218ad..80f090d7 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -38,6 +38,7 @@ abstract trait TileLinkParameters extends UsesParameters { val tlGrantTypeBits = max(log2Up(Grant.nBuiltInTypes), tlCoh.grantTypeWidth) + 1 val tlNetworkPreservesPointToPointOrdering = params(TLNetworkIsOrderedP2P) + val amoAluOperandBits = params(AmoAluOperandBits) } abstract class TLBundle extends Bundle with TileLinkParameters @@ -101,13 +102,22 @@ class Acquire extends ClientToManagerChannel M_XWR, union(opSizeOff-1, opCodeOff)) def op_size(dummy: Int = 0) = union(addrByteOff-1, opSizeOff) def addr_byte(dummy: Int = 0) = union(addrByteMSB-1, addrByteOff) - def write_mask(dummy: Int = 0) = union(tlWriteMaskBits, 1) + private def amo_offset(dummy: Int = 0) = addr_byte()(tlByteAddrBits-1, log2Up(amoAluOperandBits/8)) + def amo_shift_bits(dummy: Int = 0) = UInt(amoAluOperandBits)*amo_offset() + def wmask(dummy: Int = 0) = + Mux(isBuiltInType(Acquire.putAtomicType), + FillInterleaved(amoAluOperandBits, UIntToOH(amo_offset())), + Mux(isBuiltInType(Acquire.putBlockType) || isBuiltInType(Acquire.putType), + FillInterleaved(8, union(tlWriteMaskBits, 1)), + UInt(0, width = tlDataBits))) + def addr(dummy: Int = 0) = Cat(this.addr_block, this.addr_beat, this.addr_byte()) // Other helper funcs - def is(t: UInt) = a_type === t + def is(t: UInt) = a_type === t //TODO: make this more opaque; def ===? def isBuiltInType(dummy: Int = 0): Bool = is_builtin_type + def isBuiltInType(t: UInt): Bool = is_builtin_type && a_type === t def isSubBlockType(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesOnSubBlocks.contains(a_type) @@ -119,7 +129,7 @@ class Acquire extends ClientToManagerChannel def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && isBuiltInType() && Acquire.typesWithMultibeatData.contains(a_type) - def requiresSelfProbe(dummy: Int = 0) = Bool(false) + def requiresSelfProbe(dummy: Int = 0) = isBuiltInType()//Bool(false) def getBuiltInGrantType(dummy: Int = 0): UInt = { MuxLookup(this.a_type, Grant.putAckType, Array( @@ -229,7 +239,7 @@ object Put { addr_block: UInt, addr_beat: UInt, data: UInt, - write_mask: UInt = Acquire.fullWriteMask): Acquire = { + wmask: UInt = Acquire.fullWriteMask): Acquire = { Acquire( is_builtin_type = Bool(true), a_type = Acquire.putType, @@ -237,7 +247,7 @@ object Put { addr_beat = addr_beat, client_xact_id = client_xact_id, data = data, - union = Cat(write_mask, Bool(true))) + union = Cat(wmask, Bool(true))) } } @@ -248,7 +258,7 @@ object PutBlock { addr_block: UInt, addr_beat: UInt, data: UInt, - write_mask: UInt): Acquire = { + wmask: UInt): Acquire = { Acquire( is_builtin_type = Bool(true), a_type = Acquire.putBlockType, @@ -256,7 +266,7 @@ object PutBlock { addr_block = addr_block, addr_beat = addr_beat, data = data, - union = Cat(write_mask, (write_mask != Acquire.fullWriteMask))) + union = Cat(wmask, (wmask != Acquire.fullWriteMask))) } def apply( client_xact_id: UInt, From 40a5059cee6311f6bd5e74fc4a46aee5f34d99fe Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 00:23:11 -0700 Subject: [PATCH 315/374] clean up acquire alloc in hub --- uncore/src/main/scala/broadcast.scala | 24 ++++++++++++++---------- uncore/src/main/scala/cache.scala | 3 +-- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index c8014f45..2564140b 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -48,25 +48,29 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent when (sdq_enq) { sdq(sdq_alloc_id) := io.iacq().data } // Handle acquire transaction initiation - val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) + + val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) + alloc_arb.io.out.ready := Bool(true) + trackerAcquireIOs.zip(alloc_arb.io.in).foreach { + case(tracker, arb) => arb.valid := tracker.ready + } + val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} + val acquireMatchList = trackerList.map(_.io.has_acquire_match) val any_acquire_matches = acquireMatchList.reduce(_||_) - val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} val match_idx = Vec(acquireMatchList).indexWhere{b: Bool => b} + val acquire_idx = Mux(any_acquire_matches, match_idx, alloc_idx) - trackerAcquireIOs.zip(alloc_arb.io.in).zipWithIndex.foreach { - case((tracker, arb), i) => - arb.valid := tracker.ready + val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) + io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && !block_acquires && sdq_rdy + trackerAcquireIOs.zipWithIndex.foreach { + case(tracker, i) => tracker.bits := io.inner.acquire.bits tracker.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits - tracker.valid := arb.ready && (acquire_idx === UInt(i)) + tracker.valid := io.inner.acquire.valid && !block_acquires && (acquire_idx === UInt(i)) } - val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && - sdq_rdy && !block_acquires - alloc_arb.io.out.ready := io.inner.acquire.valid && sdq_rdy && !block_acquires // Queue to store impending Voluntary Release data val voluntary = io.irel().isVoluntary() diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index a1defb3b..4af26399 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -376,8 +376,7 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) alloc_arb.io.out.ready := Bool(true) trackerAcquireIOs.zip(alloc_arb.io.in).foreach { - case(tracker, arb) => - arb.valid := tracker.ready + case(tracker, arb) => arb.valid := tracker.ready } val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} From 672fbb574f8d34066f946680d5c34addf6d2614e Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 00:23:35 -0700 Subject: [PATCH 316/374] fix assert --- uncore/src/main/scala/cache.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 4af26399..953aa2b6 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -958,6 +958,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { //TODO: Assumes in-order network assert(!(state === s_idle && io.inner.acquire.fire() && + !io.iacq().isSubBlockType() && io.iacq().addr_beat != UInt(0)), "AcquireTracker initialized with a tail data beat.") } From b21fcdfbe0836646a2a5a13f0a5b1369dab6c7e1 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 00:24:59 -0700 Subject: [PATCH 317/374] turn off self probes again --- uncore/src/main/scala/tilelink.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 80f090d7..56dfa7db 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -129,7 +129,7 @@ class Acquire extends ClientToManagerChannel def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && isBuiltInType() && Acquire.typesWithMultibeatData.contains(a_type) - def requiresSelfProbe(dummy: Int = 0) = isBuiltInType()//Bool(false) + def requiresSelfProbe(dummy: Int = 0) = Bool(false) def getBuiltInGrantType(dummy: Int = 0): UInt = { MuxLookup(this.a_type, Grant.putAckType, Array( From 2d33dd8e6ee2dbf257a046a2619f89c75b613fee Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 01:17:44 -0700 Subject: [PATCH 318/374] pending read fix --- uncore/src/main/scala/cache.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 953aa2b6..a2e78d1b 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -782,7 +782,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { xact := io.iacq() xact.data := UInt(0) wmask_buffer.foreach { w => w := UInt(0) } - pending_reads := Mux(io.iacq().isSubBlockType(), SInt(0), SInt(-1)).toUInt + pending_reads := Mux(io.iacq().isSubBlockType(), + UIntToOH(io.iacq().addr_beat), + SInt(-1, width = innerDataBeats)).toUInt pending_writes := UInt(0) pending_resps := UInt(0) irel_had_data := Bool(false) @@ -947,9 +949,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { mergeDataPut(beat, io.iacq().wmask(), io.iacq().data) wmask_buffer(beat) := io.iacq().wmask() | wmask_buffer(beat) //iacq_data_valid(beat) := Bool(true) - pending_writes(beat) := Bool(true) + pending_writes := pending_writes | UIntToOH(io.iacq().addr_beat) } - pending_reads(beat) := Bool(true) + when(state != s_idle) { pending_reads := pending_reads | UIntToOH(io.iacq().addr_beat) } } assert(!(state != s_idle && io.inner.acquire.fire() && From fd8f6ff0269808bdc9bc2dc24a28d22e7f5890d0 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 02:10:30 -0700 Subject: [PATCH 319/374] overlapping read/resps in l2 fix --- uncore/src/main/scala/cache.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index a2e78d1b..72ef4a79 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -900,6 +900,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) pending_resps := pending_resps & ~UIntToOH(io.data.resp.bits.addr_beat) } + when(io.data.read.ready && io.data.resp.valid) { + pending_resps := (pending_resps & + ~UIntToOH(io.data.resp.bits.addr_beat)) | + UIntToOH(curr_read_beat) + } } is(s_data_resp) { when(io.data.resp.valid) { From 257dcedcb34b341197983eba6df4f64786d0e2ae Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 04:58:54 -0700 Subject: [PATCH 320/374] merge data wmask bugfix --- uncore/src/main/scala/cache.scala | 11 ++++++----- uncore/src/main/scala/tilelink.scala | 7 ++++--- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 72ef4a79..c52566c3 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -648,7 +648,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { amoalu.io.rhs := data_buffer.head // default def mergeDataPut(beat: UInt, wmask: UInt, put_data: UInt) { - data_buffer(beat) := ~wmask & data_buffer(beat) | wmask & put_data + val full = FillInterleaved(8, wmask) + data_buffer(beat) := (~full & data_buffer(beat)) | (full & put_data) + wmask_buffer(beat) := wmask | wmask_buffer(beat) } def mergeData(dataBits: Int)(beat: UInt, incoming: UInt) { @@ -656,10 +658,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val new_data = data_buffer(beat) // Newly Put data is already in the buffer amoalu.io.lhs := old_data >> xact.amo_shift_bits() amoalu.io.rhs := new_data >> xact.amo_shift_bits() - val valid_beat = (xact.is(Acquire.putBlockType) || xact.addr_beat === beat) - val wmask = Fill(dataBits, valid_beat) & wmask_buffer(beat) + val valid_beat = (xact.isBuiltInType(Acquire.putBlockType) || xact.addr_beat === beat) + val wmask = Fill(dataBits, valid_beat) & FillInterleaved(8, wmask_buffer(beat)) data_buffer(beat) := ~wmask & old_data | - wmask & Mux(xact.is(Acquire.putAtomicType), + wmask & Mux(xact.isBuiltInType(Acquire.putAtomicType), amoalu.io.out << xact.amo_shift_bits(), new_data) when(xact.is(Acquire.putAtomicType) && valid_beat) { amo_result := old_data } @@ -952,7 +954,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val beat = io.iacq().addr_beat when(io.iacq().hasData()) { mergeDataPut(beat, io.iacq().wmask(), io.iacq().data) - wmask_buffer(beat) := io.iacq().wmask() | wmask_buffer(beat) //iacq_data_valid(beat) := Bool(true) pending_writes := pending_writes | UIntToOH(io.iacq().addr_beat) } diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 56dfa7db..42c17ca4 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -106,10 +106,11 @@ class Acquire extends ClientToManagerChannel def amo_shift_bits(dummy: Int = 0) = UInt(amoAluOperandBits)*amo_offset() def wmask(dummy: Int = 0) = Mux(isBuiltInType(Acquire.putAtomicType), - FillInterleaved(amoAluOperandBits, UIntToOH(amo_offset())), + FillInterleaved(amoAluOperandBits/8, UIntToOH(amo_offset())), Mux(isBuiltInType(Acquire.putBlockType) || isBuiltInType(Acquire.putType), - FillInterleaved(8, union(tlWriteMaskBits, 1)), - UInt(0, width = tlDataBits))) + union(tlWriteMaskBits, 1), + UInt(0, width = tlWriteMaskBits))) + def full_wmask(dummy: Int = 0) = FillInterleaved(8, wmask()) def addr(dummy: Int = 0) = Cat(this.addr_block, this.addr_beat, this.addr_byte()) From ce9d4b6e70192c0353056c59dda435c2e58ab740 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 16 Mar 2015 13:27:05 -0700 Subject: [PATCH 321/374] further amo cleanups --- uncore/src/main/scala/cache.scala | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index bc12cfeb..e1c8bce4 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -572,11 +572,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val state = Reg(init=s_idle) val xact_src = Reg(io.inner.acquire.bits.header.src.clone) - val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId) })) val data_buffer = Vec.fill(innerDataBeats) { Reg(io.iacq().data.clone) } - val amo_result = Reg(io.iacq().data.clone) + val amo_result = xact.data val xact_tag_match = Reg{ Bool() } val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } @@ -638,13 +638,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { def mergeData[T <: HasTileLinkData] (byteAddrBits: Int, dataBits: Int) (buffer: Vec[UInt], beat: UInt, incoming: UInt) { - val old_data = incoming - val new_data = buffer(beat) + val old_data = incoming // Refilled, written back, or de-cached data + val new_data = buffer(beat) // Newly Put data is in the buffer val amoOpSz = UInt(amoAluOperandBits) val offset = xact.addr_byte()(byteAddrBits-1, log2Up(amoAluOperandBits/8)) amoalu.io.lhs := old_data >> offset*amoOpSz amoalu.io.rhs := new_data >> offset*amoOpSz - val valid_beat = xact.is(Acquire.putBlockType) || xact.addr_beat === beat + val valid_beat = (xact.is(Acquire.putBlockType) || xact.addr_beat === beat) && + xact.isBuiltInType() // Only custom a_types have data for now val wmask = Fill(dataBits, valid_beat) & Mux(xact.is(Acquire.putAtomicType), FillInterleaved(amoAluOperandBits, UIntToOH(offset)), @@ -746,6 +747,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.iacq().client_xact_id != xact.client_xact_id), "AcquireTracker accepted data beat from different client transaction than initial request.") + //TODO: Assumes in-order network assert(!(state === s_idle && io.inner.acquire.fire() && io.iacq().addr_beat != UInt(0)), "AcquireTracker initialized with a tail data beat.") @@ -765,6 +767,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.inner.acquire.valid) { xact_src := io.inner.acquire.bits.header.src xact := io.iacq() + xact.data := UInt(0) data_buffer(io.iacq().addr_beat) := io.iacq().data collect_iacq_data := io.iacq().hasMultibeatData() iacq_data_valid := io.iacq().hasData() << io.iacq().addr_beat From 0e51fef200776fd411b9c92e3d5eca4dc0c4d81b Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Mon, 16 Mar 2015 18:47:16 -0700 Subject: [PATCH 322/374] bugfix where an in-progress acquire can be blocked by another acquire tracker being free'd up in between --- uncore/src/main/scala/cache.scala | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index e1c8bce4..da000c5e 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -364,23 +364,28 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule (trackerList.map(_.io.incoherent) :+ wb.io.incoherent).map( _ := io.incoherent.toBits) // Handle acquire transaction initiation + val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) + val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) - val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) + alloc_arb.io.out.ready := Bool(true) + trackerAcquireIOs.zip(alloc_arb.io.in).foreach { + case(tracker, arb) => + arb.valid := tracker.ready + } + val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} + val acquireMatchList = trackerList.map(_.io.has_acquire_match) val any_acquire_matches = acquireMatchList.reduce(_||_) - val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} val match_idx = Vec(acquireMatchList).indexWhere{b: Bool => b} + val acquire_idx = Mux(any_acquire_matches, match_idx, alloc_idx) - trackerAcquireIOs.zip(alloc_arb.io.in).zipWithIndex.foreach { - case((tracker, arb), i) => - arb.valid := tracker.ready - tracker.bits := io.inner.acquire.bits - tracker.valid := arb.ready && (acquire_idx === UInt(i)) - } val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && - !block_acquires - alloc_arb.io.out.ready := io.inner.acquire.valid && !block_acquires + io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && !block_acquires + trackerAcquireIOs.zipWithIndex.foreach { + case(tracker, i) => + tracker.bits := io.inner.acquire.bits + tracker.valid := io.inner.acquire.valid && !block_acquires && (acquire_idx === UInt(i)) + } // Wire releases from clients val release_idx = Vec(trackerList.map(_.io.has_release_match) :+ From 1471d9debc04a81c98c3af32d1846bfde77cce7a Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 05:40:05 -0700 Subject: [PATCH 323/374] fix assert --- uncore/src/main/scala/cache.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index da000c5e..469adc94 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -754,6 +754,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { //TODO: Assumes in-order network assert(!(state === s_idle && io.inner.acquire.fire() && + !io.iacq().isSubBlockType() && io.iacq().addr_beat != UInt(0)), "AcquireTracker initialized with a tail data beat.") From 9bedde9a8a7317e601bf920a013ccea01b08f85e Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 12:22:57 -0700 Subject: [PATCH 324/374] re-merge mem resp queues --- uncore/src/main/scala/memserdes.scala | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 0a2b9c39..a5bea3a3 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -544,21 +544,11 @@ class MemPipeIOMemIOConverter(numRequests: Int, refillCycles: Int) extends Modul io.mem.req_data <> io.cpu.req_data // Have separate queues to allow for different mem implementations - val resp_dataq = Module((new HellaQueue(numEntries)) { new MemData }) - resp_dataq.io.enq.valid := io.mem.resp.valid - resp_dataq.io.enq.bits.data := io.mem.resp.bits.data + val resp_data_q = Module((new HellaQueue(numEntries)) { new MemResp }) + resp_data_q.io.enq <> io.mem.resp + io.cpu.resp <> resp_data_q.io.deq - val resp_tagq = Module((new HellaQueue(numEntries)) { new MemTag }) - resp_tagq.io.enq.valid := io.mem.resp.valid - resp_tagq.io.enq.bits.tag := io.mem.resp.bits.tag - - io.cpu.resp.valid := resp_dataq.io.deq.valid && resp_tagq.io.deq.valid - io.cpu.resp.bits.data := resp_dataq.io.deq.bits.data - io.cpu.resp.bits.tag := resp_tagq.io.deq.bits.tag - resp_dataq.io.deq.ready := io.cpu.resp.ready - resp_tagq.io.deq.ready := io.cpu.resp.ready - - inc := resp_dataq.io.deq.fire() + inc := resp_data_q.io.deq.fire() dec := io.mem.req_cmd.fire() && !io.mem.req_cmd.bits.rw } From 6af48e168a7e2ab8de76b67477c3619aad034b41 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 16 Mar 2015 15:06:46 -0700 Subject: [PATCH 325/374] cleanup mergeData buffer --- uncore/src/main/scala/cache.scala | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 469adc94..08ce9c3e 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -642,23 +642,23 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { def mergeData[T <: HasTileLinkData] (byteAddrBits: Int, dataBits: Int) - (buffer: Vec[UInt], beat: UInt, incoming: UInt) { + (beat: UInt, incoming: UInt) { val old_data = incoming // Refilled, written back, or de-cached data - val new_data = buffer(beat) // Newly Put data is in the buffer + val new_data = data_buffer(beat) // Newly Put data is in the buffer val amoOpSz = UInt(amoAluOperandBits) - val offset = xact.addr_byte()(byteAddrBits-1, log2Up(amoAluOperandBits/8)) - amoalu.io.lhs := old_data >> offset*amoOpSz - amoalu.io.rhs := new_data >> offset*amoOpSz + val amoOffset = xact.addr_byte()(byteAddrBits-1, log2Up(amoAluOperandBits/8)) + amoalu.io.lhs := old_data >> amoOffset*amoOpSz + amoalu.io.rhs := new_data >> amoOffset*amoOpSz val valid_beat = (xact.is(Acquire.putBlockType) || xact.addr_beat === beat) && xact.isBuiltInType() // Only custom a_types have data for now val wmask = Fill(dataBits, valid_beat) & Mux(xact.is(Acquire.putAtomicType), - FillInterleaved(amoAluOperandBits, UIntToOH(offset)), + FillInterleaved(amoAluOperandBits, UIntToOH(amoOffset)), Mux(xact.is(Acquire.putBlockType) || xact.is(Acquire.putType), - FillInterleaved(8, xact.write_mask()), + FillInterleaved(8, write_mask_buffer(beat)), UInt(0, width = dataBits))) - buffer(beat) := ~wmask & old_data | wmask & - Mux(xact.is(Acquire.putAtomicType), amoalu.io.out << offset*amoOpSz, new_data) + data_buffer(beat) := ~wmask & old_data | wmask & + Mux(xact.is(Acquire.putAtomicType), amoalu.io.out << amoOffset*amoOpSz, new_data) when(xact.is(Acquire.putAtomicType) && valid_beat) { amo_result := old_data } } val mergeDataInternal = mergeData(log2Up(rowBits/8), rowBits) _ @@ -832,7 +832,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.irel().hasData()) { irel_had_data := Bool(true) pending_coh.outer := pending_ocoh_on_irel - mergeDataInner(data_buffer, io.irel().addr_beat, io.irel().data) + mergeDataInner(io.irel().addr_beat, io.irel().data) } // We don't decrement release_count until we've received all the data beats. when(!io.irel().hasMultibeatData() || irel_data_done) { @@ -853,7 +853,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.grant.ready := Bool(true) when(io.outer.grant.valid) { when(io.ognt().hasData()) { - mergeDataOuter(data_buffer, io.ognt().addr_beat, io.ognt().data) + mergeDataOuter(io.ognt().addr_beat, io.ognt().data) ognt_had_data := Bool(true) } when(ognt_data_done) { @@ -880,13 +880,13 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { is(s_data_read) { io.data.read.valid := !collect_iacq_data || iacq_data_valid(read_data_cnt) when(io.data.resp.valid) { - mergeDataInternal(data_buffer, io.data.resp.bits.addr_beat, io.data.resp.bits.data) + mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) } when(read_data_done) { state := s_data_resp } } is(s_data_resp) { when(io.data.resp.valid) { - mergeDataInternal(data_buffer, io.data.resp.bits.addr_beat, io.data.resp.bits.data) + mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) } when(resp_data_done) { state := Mux(xact.hasData(), s_data_write, s_inner_grant) From 3f070eee1f849ff3c273c77ce732f650587241df Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 16 Mar 2015 23:41:56 -0700 Subject: [PATCH 326/374] first cut of merging puts/gets --- uncore/src/main/scala/cache.scala | 261 ++++++++++++++++----------- uncore/src/main/scala/tilelink.scala | 24 ++- 2 files changed, 170 insertions(+), 115 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 08ce9c3e..a1defb3b 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -171,6 +171,9 @@ abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgen val amoAluOperandBits = params(AmoAluOperandBits) require(amoAluOperandBits <= innerDataBits) require(rowBits == innerDataBits) // TODO: relax this by improving s_data_* states + val nSecondaryMisses = 4 + val enableGetMerging = false + val enablePutMerging = true } abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters @@ -316,6 +319,10 @@ class L2DataArray(delay: Int) extends L2HellaCacheModule { io.write.ready := Bool(true) } +class L2SecondaryMissInfo extends TLBundle + with HasTileLinkBeatId + with HasClientTransactionId + class L2HellaCacheBank(bankId: Int) extends HierarchicalCoherenceAgent with L2HellaCacheParameters { require(isPow2(nSets)) @@ -570,6 +577,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack } } + class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val io = new L2XactTrackerIO @@ -578,15 +586,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_src = Reg(io.inner.acquire.bits.header.src.clone) val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId) })) - val data_buffer = Vec.fill(innerDataBeats) { - Reg(io.iacq().data.clone) - } - val amo_result = xact.data + val data_buffer = Vec.fill(innerDataBeats){ Reg(UInt(width = innerDataBits)) } + val wmask_buffer = Vec.fill(innerDataBeats){ Reg(Bits(width = innerDataBits/8)) } val xact_tag_match = Reg{ Bool() } val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } val pending_coh = Reg{ xact_meta.coh.clone } val pending_finish = Reg{ io.outer.finish.bits.clone } + val ignt_q = Module(new Queue(new L2SecondaryMissInfo, nSecondaryMisses))(innerTLParams) val is_hit = xact_tag_match && xact_meta.coh.outer.isHit(xact.op_code()) val do_allocate = xact.allocate() @@ -594,6 +601,29 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { (xact_meta.coh.outer.requiresVoluntaryWriteback() || xact_meta.coh.inner.requiresProbesOnVoluntaryWriteback()) + val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) + val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) + val curr_probe_dst = PriorityEncoder(pending_probes) + val full_sharers = io.meta.resp.bits.meta.coh.inner.full() + val probe_self = xact.requiresSelfProbe() + val mask_self = Mux(probe_self, + full_sharers | UInt(UInt(1) << xact_src, width = nCoherentClients), + full_sharers & ~UInt(UInt(1) << xact_src, width = nCoherentClients)) + val mask_incoherent = mask_self & ~io.incoherent.toBits + + val irel_had_data = Reg(init = Bool(false)) + val ognt_had_data = Reg(init = Bool(false)) + val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) + val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire, xact.addr_beat) + val (ignt_data_idx, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) + + val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) + val pending_writes = Reg(init=Bits(0, width = innerDataBeats)) + val pending_resps = Reg(init=Bits(0, width = innerDataBeats)) + val curr_read_beat = PriorityEncoder(pending_reads) + val curr_write_beat = PriorityEncoder(pending_writes) + val pending_coh_on_hit = HierarchicalMetadata( io.meta.resp.bits.meta.coh.inner, io.meta.resp.bits.meta.coh.outer.onHit(xact.op_code())) @@ -610,29 +640,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { dst = io.inner.grant.bits.header.dst), pending_coh.outer) - val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) - val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) - val curr_p_id = PriorityEncoder(pending_probes) - val full_sharers = io.meta.resp.bits.meta.coh.inner.full() - val probe_self = xact.requiresSelfProbe() - val mask_self = Mux(probe_self, - full_sharers | UInt(UInt(1) << xact_src, width = nCoherentClients), - full_sharers & ~UInt(UInt(1) << xact_src, width = nCoherentClients)) - val mask_incoherent = mask_self & ~io.incoherent.toBits - - val collect_iacq_data = Reg(init=Bool(false)) - val iacq_data_valid = Reg(init=Bits(0, width = innerDataBeats)) - val irel_had_data = Reg(init = Bool(false)) - val ognt_had_data = Reg(init = Bool(false)) - val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) - val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) - val (ignt_data_cnt, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, xact.addr_beat) - val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire, xact.addr_beat) - val (read_data_cnt, read_data_done) = connectInternalDataBeatCounter(io.data.read, xact.addr_beat, !xact.isSubBlockType()) - val (write_data_cnt, write_data_done) = connectInternalDataBeatCounter(io.data.write, xact.addr_beat, !xact.isSubBlockType() || ognt_had_data || irel_had_data) - val resp_data_done = connectInternalDataBeatCounter(io.data.resp, !xact.isSubBlockType()) - + val amo_result = xact.data val amoalu = Module(new AMOALU) amoalu.io.addr := xact.addr() amoalu.io.cmd := xact.op_code() @@ -640,42 +648,62 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { amoalu.io.lhs := io.data.resp.bits.data //default amoalu.io.rhs := data_buffer.head // default - def mergeData[T <: HasTileLinkData] - (byteAddrBits: Int, dataBits: Int) - (beat: UInt, incoming: UInt) { + def mergeDataPut(beat: UInt, wmask: UInt, put_data: UInt) { + data_buffer(beat) := ~wmask & data_buffer(beat) | wmask & put_data + } + + def mergeData(dataBits: Int)(beat: UInt, incoming: UInt) { val old_data = incoming // Refilled, written back, or de-cached data - val new_data = data_buffer(beat) // Newly Put data is in the buffer - val amoOpSz = UInt(amoAluOperandBits) - val amoOffset = xact.addr_byte()(byteAddrBits-1, log2Up(amoAluOperandBits/8)) - amoalu.io.lhs := old_data >> amoOffset*amoOpSz - amoalu.io.rhs := new_data >> amoOffset*amoOpSz - val valid_beat = (xact.is(Acquire.putBlockType) || xact.addr_beat === beat) && - xact.isBuiltInType() // Only custom a_types have data for now - val wmask = Fill(dataBits, valid_beat) & - Mux(xact.is(Acquire.putAtomicType), - FillInterleaved(amoAluOperandBits, UIntToOH(amoOffset)), - Mux(xact.is(Acquire.putBlockType) || xact.is(Acquire.putType), - FillInterleaved(8, write_mask_buffer(beat)), - UInt(0, width = dataBits))) - data_buffer(beat) := ~wmask & old_data | wmask & - Mux(xact.is(Acquire.putAtomicType), amoalu.io.out << amoOffset*amoOpSz, new_data) + val new_data = data_buffer(beat) // Newly Put data is already in the buffer + amoalu.io.lhs := old_data >> xact.amo_shift_bits() + amoalu.io.rhs := new_data >> xact.amo_shift_bits() + val valid_beat = (xact.is(Acquire.putBlockType) || xact.addr_beat === beat) + val wmask = Fill(dataBits, valid_beat) & wmask_buffer(beat) + data_buffer(beat) := ~wmask & old_data | + wmask & Mux(xact.is(Acquire.putAtomicType), + amoalu.io.out << xact.amo_shift_bits(), + new_data) when(xact.is(Acquire.putAtomicType) && valid_beat) { amo_result := old_data } } - val mergeDataInternal = mergeData(log2Up(rowBits/8), rowBits) _ - val mergeDataInner = mergeData(innerByteAddrBits, innerDataBits) _ - val mergeDataOuter = mergeData(outerByteAddrBits, outerDataBits) _ + val mergeDataInternal = mergeData(rowBits) _ + val mergeDataInner = mergeData(innerDataBits) _ + val mergeDataOuter = mergeData(outerDataBits) _ + + val can_merge_iacq_get = Bool(enableGetMerging) && + (xact.isBuiltInType(Acquire.getType) && + io.iacq().isBuiltInType(Acquire.getType)) && + (xact_src === io.inner.acquire.bits.header.src) && + xact.conflicts(io.iacq()) && + Vec(s_meta_read, s_meta_resp, s_wb_req, s_wb_resp, + s_probe, s_outer_acquire, s_outer_grant, + s_outer_finish).contains(state) && + do_allocate && + ignt_q.io.enq.ready + //TODO: mix Puts and PutBlocks + val can_merge_iacq_put = ((Bool(enablePutMerging) && + (xact.isBuiltInType(Acquire.putType) && + io.iacq().isBuiltInType(Acquire.putType))) || + (xact.isBuiltInType(Acquire.putBlockType) && + io.iacq().isBuiltInType(Acquire.putBlockType))) && + (xact_src === io.inner.acquire.bits.header.src) && + (xact.client_xact_id === io.iacq().client_xact_id) && + xact.conflicts(io.iacq()) && + Vec(s_meta_read, s_meta_resp, s_wb_req, s_wb_resp, + s_probe, s_outer_acquire, s_outer_grant, + s_outer_finish, s_data_read, + s_data_resp).contains(state) && + do_allocate && + ignt_q.io.enq.ready - //TODO: Allow hit under miss for stores val in_same_set = xact.addr_block(idxMSB,idxLSB) === io.iacq().addr_block(idxMSB,idxLSB) - io.has_acquire_conflict := (xact.conflicts(io.iacq()) || in_same_set) && - (state != s_idle) && - !collect_iacq_data - io.has_acquire_match := xact.conflicts(io.iacq()) && - collect_iacq_data io.has_release_match := xact.conflicts(io.irel()) && !io.irel().isVoluntary() && (state === s_probe) + io.has_acquire_match := can_merge_iacq_put || can_merge_iacq_get + io.has_acquire_conflict := (xact.conflicts(io.iacq()) || in_same_set) && + (state != s_idle) && + !io.has_acquire_match // If we're allocating in this cache, we can use the current metadata // to make an appropriate custom Acquire, otherwise we copy over the @@ -697,21 +725,25 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_p_id + io.inner.probe.bits.header.dst := curr_probe_dst io.inner.probe.bits.payload := pending_coh.inner.makeProbe(xact) - io.inner.grant.valid := Bool(false) + io.inner.grant.valid := state === s_inner_grant && ignt_q.io.deq.valid io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := xact_src io.inner.grant.bits.payload := pending_coh.inner.makeGrant( acq = xact, manager_xact_id = UInt(trackerId), - addr_beat = ignt_data_cnt, + addr_beat = ignt_data_idx, data = Mux(xact.is(Acquire.putAtomicType), amo_result, - data_buffer(ignt_data_cnt))) + data_buffer(ignt_data_idx))) + io.ignt().client_xact_id := ignt_q.io.deq.bits.client_xact_id + ignt_q.io.deq.ready := ignt_data_done - io.inner.acquire.ready := Bool(false) + io.inner.acquire.ready := state === s_idle || + can_merge_iacq_put || + can_merge_iacq_get io.inner.release.ready := Bool(false) io.inner.finish.ready := Bool(false) @@ -719,14 +751,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.data.read.bits.id := UInt(trackerId) io.data.read.bits.way_en := xact_way_en io.data.read.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) - io.data.read.bits.addr_beat := read_data_cnt + io.data.read.bits.addr_beat := curr_read_beat io.data.write.valid := Bool(false) io.data.write.bits.id := UInt(trackerId) io.data.write.bits.way_en := xact_way_en io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) - io.data.write.bits.addr_beat := write_data_cnt + io.data.write.bits.addr_beat := curr_write_beat io.data.write.bits.wmask := SInt(-1) - io.data.write.bits.data := data_buffer(write_data_cnt) + io.data.write.bits.data := data_buffer(curr_write_beat) io.meta.read.valid := Bool(false) io.meta.read.bits.id := UInt(trackerId) io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) @@ -744,39 +776,16 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.wb.req.bits.way_en := xact_way_en io.wb.req.bits.id := UInt(trackerId) - assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && - io.inner.acquire.bits.header.src != xact_src), - "AcquireTracker accepted data beat from different network source than initial request.") - - assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && - io.iacq().client_xact_id != xact.client_xact_id), - "AcquireTracker accepted data beat from different client transaction than initial request.") - - //TODO: Assumes in-order network - assert(!(state === s_idle && io.inner.acquire.fire() && - !io.iacq().isSubBlockType() && - io.iacq().addr_beat != UInt(0)), - "AcquireTracker initialized with a tail data beat.") - - when(collect_iacq_data) { - io.inner.acquire.ready := Bool(true) - when(io.inner.acquire.valid) { - data_buffer(io.iacq().addr_beat) := io.iacq().data - iacq_data_valid(io.iacq().addr_beat) := Bool(true) - } - when(iacq_data_done) { collect_iacq_data := Bool(false) } - } - switch (state) { is(s_idle) { - io.inner.acquire.ready := Bool(true) when(io.inner.acquire.valid) { xact_src := io.inner.acquire.bits.header.src xact := io.iacq() xact.data := UInt(0) - data_buffer(io.iacq().addr_beat) := io.iacq().data - collect_iacq_data := io.iacq().hasMultibeatData() - iacq_data_valid := io.iacq().hasData() << io.iacq().addr_beat + wmask_buffer.foreach { w => w := UInt(0) } + pending_reads := Mux(io.iacq().isSubBlockType(), SInt(0), SInt(-1)).toUInt + pending_writes := UInt(0) + pending_resps := UInt(0) irel_had_data := Bool(false) ognt_had_data := Bool(false) state := s_meta_read @@ -820,7 +829,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { // Send probes io.inner.probe.valid := pending_probes != UInt(0) when(io.inner.probe.ready) { - pending_probes := pending_probes & ~UIntToOH(curr_p_id) + pending_probes := pending_probes & ~UIntToOH(curr_probe_dst) } // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) @@ -830,9 +839,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { //TODO: make sure cacq data is actually present before accpeting // release data to merge! when(io.irel().hasData()) { - irel_had_data := Bool(true) pending_coh.outer := pending_ocoh_on_irel mergeDataInner(io.irel().addr_beat, io.irel().data) + pending_writes := pending_writes | UIntToOH(io.irel().addr_beat) } // We don't decrement release_count until we've received all the data beats. when(!io.irel().hasMultibeatData() || irel_data_done) { @@ -840,11 +849,13 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } } when(release_count === UInt(0)) { - state := Mux(is_hit, Mux(irel_had_data, s_data_write, s_data_read), s_outer_acquire) + state := Mux(is_hit, + Mux(pending_writes.orR, s_data_write, s_data_read), + s_outer_acquire) } } is(s_outer_acquire) { - io.outer.acquire.valid := !iacq_data_done // collect all data before refilling + io.outer.acquire.valid := Bool(true) when(oacq_data_done) { state := s_outer_grant } @@ -854,7 +865,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.outer.grant.valid) { when(io.ognt().hasData()) { mergeDataOuter(io.ognt().addr_beat, io.ognt().data) - ognt_had_data := Bool(true) + pending_writes := pending_writes | UIntToOH(io.ognt().addr_beat) } when(ognt_data_done) { pending_coh := pending_coh_on_ognt @@ -865,7 +876,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { state := s_outer_finish }.otherwise { state := Mux(!do_allocate, s_inner_grant, - Mux(io.ognt().hasData(), s_data_write, s_data_read)) + Mux(pending_writes.orR, s_data_write, s_data_read)) } } } @@ -874,31 +885,39 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.finish.valid := Bool(true) when(io.outer.finish.ready) { state := Mux(!do_allocate, s_inner_grant, - Mux(ognt_had_data, s_data_write, s_data_read)) + Mux(pending_writes.orR, s_data_write, s_data_read)) } } is(s_data_read) { - io.data.read.valid := !collect_iacq_data || iacq_data_valid(read_data_cnt) + io.data.read.valid := pending_reads.orR + when(io.data.read.ready) { + pending_resps := pending_resps | UIntToOH(curr_read_beat) + pending_reads := pending_reads & ~UIntToOH(curr_read_beat) + when(PopCount(pending_reads) <= UInt(1)) { state := s_data_resp } + } when(io.data.resp.valid) { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) + pending_resps := pending_resps & ~UIntToOH(io.data.resp.bits.addr_beat) } - when(read_data_done) { state := s_data_resp } } is(s_data_resp) { when(io.data.resp.valid) { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) - } - when(resp_data_done) { - state := Mux(xact.hasData(), s_data_write, s_inner_grant) + pending_resps := pending_resps & ~UIntToOH(io.data.resp.bits.addr_beat) + when(PopCount(pending_resps) <= UInt(1)) { + state := Mux(pending_writes.orR, s_data_write, s_inner_grant) + } } } is(s_data_write) { - io.data.write.valid := Bool(true) - when(write_data_done) { state := s_inner_grant } + io.data.write.valid := pending_writes.orR //TODO make sure all acquire data is present + when(io.data.write.ready) { + pending_writes := pending_writes & ~UIntToOH(curr_write_beat) + when(PopCount(pending_writes) <= UInt(1)) { state := s_inner_grant } + } } is(s_inner_grant) { - io.inner.grant.valid := Bool(true) - when(ignt_data_done) { + when(ignt_data_done && ignt_q.io.count === UInt(1)) { val meta_dirty = !is_hit || pending_coh_on_ignt != xact_meta.coh when(meta_dirty) { pending_coh := pending_coh_on_ignt } state := Mux(meta_dirty, s_meta_write, @@ -916,6 +935,32 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.inner.finish.valid) { state := s_idle } } } + + ignt_q.io.enq.valid := io.inner.acquire.fire() && + (state === s_idle || !xact.hasMultibeatData()) + ignt_q.io.enq.bits.client_xact_id := io.iacq().client_xact_id + ignt_q.io.enq.bits.addr_beat := io.iacq().addr_beat + + // Handle Get and Put merging + when(io.inner.acquire.fire()) { + val beat = io.iacq().addr_beat + when(io.iacq().hasData()) { + mergeDataPut(beat, io.iacq().wmask(), io.iacq().data) + wmask_buffer(beat) := io.iacq().wmask() | wmask_buffer(beat) + //iacq_data_valid(beat) := Bool(true) + pending_writes(beat) := Bool(true) + } + pending_reads(beat) := Bool(true) + } + + assert(!(state != s_idle && io.inner.acquire.fire() && + io.inner.acquire.bits.header.src != xact_src), + "AcquireTracker accepted data beat from different network source than initial request.") + + //TODO: Assumes in-order network + assert(!(state === s_idle && io.inner.acquire.fire() && + io.iacq().addr_beat != UInt(0)), + "AcquireTracker initialized with a tail data beat.") } class L2WritebackReq extends L2HellaCacheBundle @@ -953,7 +998,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val irel_had_data = Reg(init = Bool(false)) val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) - val curr_p_id = PriorityEncoder(pending_probes) + val curr_probe_dst = PriorityEncoder(pending_probes) val full_sharers = io.wb.req.bits.coh.inner.full() val mask_incoherent = full_sharers & ~io.incoherent.toBits @@ -989,7 +1034,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_p_id + io.inner.probe.bits.header.dst := curr_probe_dst io.inner.probe.bits.payload := xact_coh.inner.makeProbeForVoluntaryWriteback(xact_addr_block) @@ -1030,7 +1075,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { // Send probes io.inner.probe.valid := pending_probes != UInt(0) when(io.inner.probe.ready) { - pending_probes := pending_probes & ~UIntToOH(curr_p_id) + pending_probes := pending_probes & ~UIntToOH(curr_probe_dst) } // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 7bd218ad..80f090d7 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -38,6 +38,7 @@ abstract trait TileLinkParameters extends UsesParameters { val tlGrantTypeBits = max(log2Up(Grant.nBuiltInTypes), tlCoh.grantTypeWidth) + 1 val tlNetworkPreservesPointToPointOrdering = params(TLNetworkIsOrderedP2P) + val amoAluOperandBits = params(AmoAluOperandBits) } abstract class TLBundle extends Bundle with TileLinkParameters @@ -101,13 +102,22 @@ class Acquire extends ClientToManagerChannel M_XWR, union(opSizeOff-1, opCodeOff)) def op_size(dummy: Int = 0) = union(addrByteOff-1, opSizeOff) def addr_byte(dummy: Int = 0) = union(addrByteMSB-1, addrByteOff) - def write_mask(dummy: Int = 0) = union(tlWriteMaskBits, 1) + private def amo_offset(dummy: Int = 0) = addr_byte()(tlByteAddrBits-1, log2Up(amoAluOperandBits/8)) + def amo_shift_bits(dummy: Int = 0) = UInt(amoAluOperandBits)*amo_offset() + def wmask(dummy: Int = 0) = + Mux(isBuiltInType(Acquire.putAtomicType), + FillInterleaved(amoAluOperandBits, UIntToOH(amo_offset())), + Mux(isBuiltInType(Acquire.putBlockType) || isBuiltInType(Acquire.putType), + FillInterleaved(8, union(tlWriteMaskBits, 1)), + UInt(0, width = tlDataBits))) + def addr(dummy: Int = 0) = Cat(this.addr_block, this.addr_beat, this.addr_byte()) // Other helper funcs - def is(t: UInt) = a_type === t + def is(t: UInt) = a_type === t //TODO: make this more opaque; def ===? def isBuiltInType(dummy: Int = 0): Bool = is_builtin_type + def isBuiltInType(t: UInt): Bool = is_builtin_type && a_type === t def isSubBlockType(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesOnSubBlocks.contains(a_type) @@ -119,7 +129,7 @@ class Acquire extends ClientToManagerChannel def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && isBuiltInType() && Acquire.typesWithMultibeatData.contains(a_type) - def requiresSelfProbe(dummy: Int = 0) = Bool(false) + def requiresSelfProbe(dummy: Int = 0) = isBuiltInType()//Bool(false) def getBuiltInGrantType(dummy: Int = 0): UInt = { MuxLookup(this.a_type, Grant.putAckType, Array( @@ -229,7 +239,7 @@ object Put { addr_block: UInt, addr_beat: UInt, data: UInt, - write_mask: UInt = Acquire.fullWriteMask): Acquire = { + wmask: UInt = Acquire.fullWriteMask): Acquire = { Acquire( is_builtin_type = Bool(true), a_type = Acquire.putType, @@ -237,7 +247,7 @@ object Put { addr_beat = addr_beat, client_xact_id = client_xact_id, data = data, - union = Cat(write_mask, Bool(true))) + union = Cat(wmask, Bool(true))) } } @@ -248,7 +258,7 @@ object PutBlock { addr_block: UInt, addr_beat: UInt, data: UInt, - write_mask: UInt): Acquire = { + wmask: UInt): Acquire = { Acquire( is_builtin_type = Bool(true), a_type = Acquire.putBlockType, @@ -256,7 +266,7 @@ object PutBlock { addr_block = addr_block, addr_beat = addr_beat, data = data, - union = Cat(write_mask, (write_mask != Acquire.fullWriteMask))) + union = Cat(wmask, (wmask != Acquire.fullWriteMask))) } def apply( client_xact_id: UInt, From 6d565d22e3e991ecb6abdfa75b1d19053024dbb3 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 00:23:11 -0700 Subject: [PATCH 327/374] clean up acquire alloc in hub --- uncore/src/main/scala/broadcast.scala | 24 ++++++++++++++---------- uncore/src/main/scala/cache.scala | 3 +-- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index c8014f45..2564140b 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -48,25 +48,29 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent when (sdq_enq) { sdq(sdq_alloc_id) := io.iacq().data } // Handle acquire transaction initiation - val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) + + val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) + alloc_arb.io.out.ready := Bool(true) + trackerAcquireIOs.zip(alloc_arb.io.in).foreach { + case(tracker, arb) => arb.valid := tracker.ready + } + val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} + val acquireMatchList = trackerList.map(_.io.has_acquire_match) val any_acquire_matches = acquireMatchList.reduce(_||_) - val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} val match_idx = Vec(acquireMatchList).indexWhere{b: Bool => b} + val acquire_idx = Mux(any_acquire_matches, match_idx, alloc_idx) - trackerAcquireIOs.zip(alloc_arb.io.in).zipWithIndex.foreach { - case((tracker, arb), i) => - arb.valid := tracker.ready + val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) + io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && !block_acquires && sdq_rdy + trackerAcquireIOs.zipWithIndex.foreach { + case(tracker, i) => tracker.bits := io.inner.acquire.bits tracker.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits - tracker.valid := arb.ready && (acquire_idx === UInt(i)) + tracker.valid := io.inner.acquire.valid && !block_acquires && (acquire_idx === UInt(i)) } - val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && - sdq_rdy && !block_acquires - alloc_arb.io.out.ready := io.inner.acquire.valid && sdq_rdy && !block_acquires // Queue to store impending Voluntary Release data val voluntary = io.irel().isVoluntary() diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index a1defb3b..4af26399 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -376,8 +376,7 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) alloc_arb.io.out.ready := Bool(true) trackerAcquireIOs.zip(alloc_arb.io.in).foreach { - case(tracker, arb) => - arb.valid := tracker.ready + case(tracker, arb) => arb.valid := tracker.ready } val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} From d9598d26f250ec6863ca8d1c82956287848b1e9b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 00:23:35 -0700 Subject: [PATCH 328/374] fix assert --- uncore/src/main/scala/cache.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 4af26399..953aa2b6 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -958,6 +958,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { //TODO: Assumes in-order network assert(!(state === s_idle && io.inner.acquire.fire() && + !io.iacq().isSubBlockType() && io.iacq().addr_beat != UInt(0)), "AcquireTracker initialized with a tail data beat.") } From 23f8033df55c59a85422b4213ed2bebc24f72437 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 00:24:59 -0700 Subject: [PATCH 329/374] turn off self probes again --- uncore/src/main/scala/tilelink.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 80f090d7..56dfa7db 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -129,7 +129,7 @@ class Acquire extends ClientToManagerChannel def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && isBuiltInType() && Acquire.typesWithMultibeatData.contains(a_type) - def requiresSelfProbe(dummy: Int = 0) = isBuiltInType()//Bool(false) + def requiresSelfProbe(dummy: Int = 0) = Bool(false) def getBuiltInGrantType(dummy: Int = 0): UInt = { MuxLookup(this.a_type, Grant.putAckType, Array( From 730a13abf27d9fc15902b23970097f90860acfd7 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 01:17:44 -0700 Subject: [PATCH 330/374] pending read fix --- uncore/src/main/scala/cache.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 953aa2b6..a2e78d1b 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -782,7 +782,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { xact := io.iacq() xact.data := UInt(0) wmask_buffer.foreach { w => w := UInt(0) } - pending_reads := Mux(io.iacq().isSubBlockType(), SInt(0), SInt(-1)).toUInt + pending_reads := Mux(io.iacq().isSubBlockType(), + UIntToOH(io.iacq().addr_beat), + SInt(-1, width = innerDataBeats)).toUInt pending_writes := UInt(0) pending_resps := UInt(0) irel_had_data := Bool(false) @@ -947,9 +949,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { mergeDataPut(beat, io.iacq().wmask(), io.iacq().data) wmask_buffer(beat) := io.iacq().wmask() | wmask_buffer(beat) //iacq_data_valid(beat) := Bool(true) - pending_writes(beat) := Bool(true) + pending_writes := pending_writes | UIntToOH(io.iacq().addr_beat) } - pending_reads(beat) := Bool(true) + when(state != s_idle) { pending_reads := pending_reads | UIntToOH(io.iacq().addr_beat) } } assert(!(state != s_idle && io.inner.acquire.fire() && From dc8861409449c6e1560da3d6389ddc1d71cfbdef Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 02:10:30 -0700 Subject: [PATCH 331/374] overlapping read/resps in l2 fix --- uncore/src/main/scala/cache.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index a2e78d1b..72ef4a79 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -900,6 +900,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) pending_resps := pending_resps & ~UIntToOH(io.data.resp.bits.addr_beat) } + when(io.data.read.ready && io.data.resp.valid) { + pending_resps := (pending_resps & + ~UIntToOH(io.data.resp.bits.addr_beat)) | + UIntToOH(curr_read_beat) + } } is(s_data_resp) { when(io.data.resp.valid) { From 5c2461c74339da1698fcb08d539c95d8be5b122b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 04:58:54 -0700 Subject: [PATCH 332/374] merge data wmask bugfix --- uncore/src/main/scala/cache.scala | 11 ++++++----- uncore/src/main/scala/tilelink.scala | 7 ++++--- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 72ef4a79..c52566c3 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -648,7 +648,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { amoalu.io.rhs := data_buffer.head // default def mergeDataPut(beat: UInt, wmask: UInt, put_data: UInt) { - data_buffer(beat) := ~wmask & data_buffer(beat) | wmask & put_data + val full = FillInterleaved(8, wmask) + data_buffer(beat) := (~full & data_buffer(beat)) | (full & put_data) + wmask_buffer(beat) := wmask | wmask_buffer(beat) } def mergeData(dataBits: Int)(beat: UInt, incoming: UInt) { @@ -656,10 +658,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val new_data = data_buffer(beat) // Newly Put data is already in the buffer amoalu.io.lhs := old_data >> xact.amo_shift_bits() amoalu.io.rhs := new_data >> xact.amo_shift_bits() - val valid_beat = (xact.is(Acquire.putBlockType) || xact.addr_beat === beat) - val wmask = Fill(dataBits, valid_beat) & wmask_buffer(beat) + val valid_beat = (xact.isBuiltInType(Acquire.putBlockType) || xact.addr_beat === beat) + val wmask = Fill(dataBits, valid_beat) & FillInterleaved(8, wmask_buffer(beat)) data_buffer(beat) := ~wmask & old_data | - wmask & Mux(xact.is(Acquire.putAtomicType), + wmask & Mux(xact.isBuiltInType(Acquire.putAtomicType), amoalu.io.out << xact.amo_shift_bits(), new_data) when(xact.is(Acquire.putAtomicType) && valid_beat) { amo_result := old_data } @@ -952,7 +954,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val beat = io.iacq().addr_beat when(io.iacq().hasData()) { mergeDataPut(beat, io.iacq().wmask(), io.iacq().data) - wmask_buffer(beat) := io.iacq().wmask() | wmask_buffer(beat) //iacq_data_valid(beat) := Bool(true) pending_writes := pending_writes | UIntToOH(io.iacq().addr_beat) } diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 56dfa7db..42c17ca4 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -106,10 +106,11 @@ class Acquire extends ClientToManagerChannel def amo_shift_bits(dummy: Int = 0) = UInt(amoAluOperandBits)*amo_offset() def wmask(dummy: Int = 0) = Mux(isBuiltInType(Acquire.putAtomicType), - FillInterleaved(amoAluOperandBits, UIntToOH(amo_offset())), + FillInterleaved(amoAluOperandBits/8, UIntToOH(amo_offset())), Mux(isBuiltInType(Acquire.putBlockType) || isBuiltInType(Acquire.putType), - FillInterleaved(8, union(tlWriteMaskBits, 1)), - UInt(0, width = tlDataBits))) + union(tlWriteMaskBits, 1), + UInt(0, width = tlWriteMaskBits))) + def full_wmask(dummy: Int = 0) = FillInterleaved(8, wmask()) def addr(dummy: Int = 0) = Cat(this.addr_block, this.addr_beat, this.addr_byte()) From fc0ae81a977b096bd2b94d6b2b031f206341a13f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 15:04:30 -0700 Subject: [PATCH 333/374] added finish counter --- uncore/src/main/scala/cache.scala | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index c52566c3..d4dbd75f 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -610,12 +610,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { full_sharers & ~UInt(UInt(1) << xact_src, width = nCoherentClients)) val mask_incoherent = mask_self & ~io.incoherent.toBits - val irel_had_data = Reg(init = Bool(false)) - val ognt_had_data = Reg(init = Bool(false)) val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire, xact.addr_beat) val (ignt_data_idx, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) + val ifin_cnt = Reg(init = UInt(0, width = log2Up(nSecondaryMisses+1))) + when(ignt_data_done) { ifin_cnt := ifin_cnt + UInt(1) } val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) val pending_writes = Reg(init=Bits(0, width = innerDataBeats)) @@ -789,8 +789,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { SInt(-1, width = innerDataBeats)).toUInt pending_writes := UInt(0) pending_resps := UInt(0) - irel_had_data := Bool(false) - ognt_had_data := Bool(false) + ifin_cnt := UInt(0) state := s_meta_read } } @@ -940,7 +939,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } is(s_inner_finish) { io.inner.finish.ready := Bool(true) - when(io.inner.finish.valid) { state := s_idle } + when(io.inner.finish.valid) { + ifin_cnt := ifin_cnt - UInt(1) + when(ifin_cnt <= UInt(1)) { state := s_idle } + } } } From a52a729ab90bac736e6f5235518b475be26bb18a Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 17 Mar 2015 15:54:21 -0700 Subject: [PATCH 334/374] bugfix wmask handling --- uncore/src/main/scala/cache.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index d4dbd75f..ed8d7051 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -658,13 +658,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val new_data = data_buffer(beat) // Newly Put data is already in the buffer amoalu.io.lhs := old_data >> xact.amo_shift_bits() amoalu.io.rhs := new_data >> xact.amo_shift_bits() - val valid_beat = (xact.isBuiltInType(Acquire.putBlockType) || xact.addr_beat === beat) - val wmask = Fill(dataBits, valid_beat) & FillInterleaved(8, wmask_buffer(beat)) + val wmask = FillInterleaved(8, wmask_buffer(beat)) data_buffer(beat) := ~wmask & old_data | wmask & Mux(xact.isBuiltInType(Acquire.putAtomicType), amoalu.io.out << xact.amo_shift_bits(), new_data) - when(xact.is(Acquire.putAtomicType) && valid_beat) { amo_result := old_data } + when(xact.is(Acquire.putAtomicType) && xact.addr_beat === beat) { amo_result := old_data } } val mergeDataInternal = mergeData(rowBits) _ val mergeDataInner = mergeData(innerDataBits) _ From 4fd01d82b86fffdca3364e1c47546311d66ef8e8 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 17 Mar 2015 17:48:02 -0700 Subject: [PATCH 335/374] don't block finish messages when grant'ing --- uncore/src/main/scala/cache.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index ed8d7051..cd68a8df 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -929,6 +929,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { state := Mux(meta_dirty, s_meta_write, Mux(io.ignt().requiresAck(), s_inner_finish, s_idle)) } + io.inner.finish.ready := Bool(true) + when(io.inner.finish.valid) { + ifin_cnt := ifin_cnt - Mux(ignt_data_done, UInt(0), UInt(1)) + } } is(s_meta_write) { io.meta.write.valid := Bool(true) @@ -942,6 +946,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { ifin_cnt := ifin_cnt - UInt(1) when(ifin_cnt <= UInt(1)) { state := s_idle } } + when(ifin_cnt === UInt(0)) { state := s_idle } } } From b08dced37cbe61589355c8505a4403544d35f311 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 17:51:00 -0700 Subject: [PATCH 336/374] first cut at pending scoreboarding --- uncore/src/main/scala/cache.scala | 90 +++++++++++++++++------------- uncore/src/main/scala/uncore.scala | 7 +++ 2 files changed, 57 insertions(+), 40 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index cd68a8df..0680442d 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -446,17 +446,28 @@ abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { (Mux(!full_block, beat, multi_cnt), Mux(!full_block, inc, multi_done)) } else { (UInt(0), inc) } } + def connectInternalDataBeatCounter[T <: HasL2BeatAddr]( in: DecoupledIO[T], beat: UInt = UInt(0), full_block: Bool = Bool(true)) = { connectDataBeatCounter(in.fire(), in.bits, beat, full_block) } + def connectInternalDataBeatCounter[T <: HasL2Data]( in: ValidIO[T], full_block: Bool = Bool(true)) = { connectDataBeatCounter(in.valid, in.bits, UInt(0), full_block)._2 } + + def addInternalPendingBit[T <: HasL2BeatAddr](in: DecoupledIO[T]) = + Fill(in.bits.refillCycles, in.fire()) & UIntToOH(in.bits.addr_beat) + + def dropPendingBit[T <: HasL2BeatAddr] (in: DecoupledIO[T]) = + Fill(in.bits.refillCycles, in.fire()) & ~UIntToOH(in.bits.addr_beat) + + def dropInternalPendingBit[T <: HasL2BeatAddr] (in: ValidIO[T]) = + Fill(in.bits.refillCycles, in.valid) & ~UIntToOH(in.bits.addr_beat) } class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTracker { @@ -591,8 +602,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } val pending_coh = Reg{ xact_meta.coh.clone } - val pending_finish = Reg{ io.outer.finish.bits.clone } - val ignt_q = Module(new Queue(new L2SecondaryMissInfo, nSecondaryMisses))(innerTLParams) val is_hit = xact_tag_match && xact_meta.coh.outer.isHit(xact.op_code()) val do_allocate = xact.allocate() @@ -611,18 +620,38 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val mask_incoherent = mask_self & ~io.incoherent.toBits val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire, xact.addr_beat) + val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) + val pending_ofin = Reg{ io.outer.finish.bits.clone } + + val ignt_q = Module(new Queue(new L2SecondaryMissInfo, nSecondaryMisses))(innerTLParams) val (ignt_data_idx, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) + ignt_q.io.enq.valid := Bool(false) + ignt_q.io.enq.bits.client_xact_id := io.iacq().client_xact_id + ignt_q.io.enq.bits.addr_beat := io.iacq().addr_beat + ignt_q.io.deq.ready := ignt_data_done + val ifin_cnt = Reg(init = UInt(0, width = log2Up(nSecondaryMisses+1))) when(ignt_data_done) { ifin_cnt := ifin_cnt + UInt(1) } - val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) - val pending_writes = Reg(init=Bits(0, width = innerDataBeats)) - val pending_resps = Reg(init=Bits(0, width = innerDataBeats)) + pending_reads := pending_reads | + addPendingBit(io.inner.acquire) | + dropPendingBit(io.data.read) val curr_read_beat = PriorityEncoder(pending_reads) + + val pending_writes = Reg(init=Bits(0, width = innerDataBeats)) + pending_writes := pending_writes | + addPendingBit(io.inner.acquire) | + addPendingBit(io.inner.release) | + addPendingBit(io.outer.grant) & + dropPendingBit(io.data.write) val curr_write_beat = PriorityEncoder(pending_writes) + val pending_resps = Reg(init=Bits(0, width = innerDataBeats)) + pending_resps := pending_resps | + addInternalPendingBit(io.data.read) & + dropInternalPendingBit(io.data.resp) + val pending_coh_on_hit = HierarchicalMetadata( io.meta.resp.bits.meta.coh.inner, io.meta.resp.bits.meta.coh.outer.onHit(xact.op_code())) @@ -720,8 +749,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.release.valid := Bool(false) io.outer.grant.ready := Bool(false) io.outer.finish.valid := Bool(false) - io.outer.finish.bits := pending_finish - val pending_finish_on_ognt = io.ognt().makeFinish() + io.outer.finish.bits := pending_ofin + val pending_ofin_on_ognt = io.ognt().makeFinish() io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) @@ -739,7 +768,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { amo_result, data_buffer(ignt_data_idx))) io.ignt().client_xact_id := ignt_q.io.deq.bits.client_xact_id - ignt_q.io.deq.ready := ignt_data_done io.inner.acquire.ready := state === s_idle || can_merge_iacq_put || @@ -789,6 +817,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_writes := UInt(0) pending_resps := UInt(0) ifin_cnt := UInt(0) + ignt_q.io.enq.valid := Bool(true) state := s_meta_read } } @@ -842,7 +871,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.irel().hasData()) { pending_coh.outer := pending_ocoh_on_irel mergeDataInner(io.irel().addr_beat, io.irel().data) - pending_writes := pending_writes | UIntToOH(io.irel().addr_beat) } // We don't decrement release_count until we've received all the data beats. when(!io.irel().hasMultibeatData() || irel_data_done) { @@ -866,14 +894,13 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.outer.grant.valid) { when(io.ognt().hasData()) { mergeDataOuter(io.ognt().addr_beat, io.ognt().data) - pending_writes := pending_writes | UIntToOH(io.ognt().addr_beat) } when(ognt_data_done) { pending_coh := pending_coh_on_ognt when(io.ognt().requiresAck()) { - pending_finish.payload := pending_finish_on_ognt - pending_finish.header.dst := io.outer.grant.bits.header.src - pending_finish.header.src := UInt(bankId) + pending_ofin.payload := pending_ofin_on_ognt + pending_ofin.header.dst := io.outer.grant.bits.header.src + pending_ofin.header.src := UInt(bankId) state := s_outer_finish }.otherwise { state := Mux(!do_allocate, s_inner_grant, @@ -892,18 +919,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { is(s_data_read) { io.data.read.valid := pending_reads.orR when(io.data.read.ready) { - pending_resps := pending_resps | UIntToOH(curr_read_beat) - pending_reads := pending_reads & ~UIntToOH(curr_read_beat) when(PopCount(pending_reads) <= UInt(1)) { state := s_data_resp } } when(io.data.resp.valid) { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) - pending_resps := pending_resps & ~UIntToOH(io.data.resp.bits.addr_beat) - } - when(io.data.read.ready && io.data.resp.valid) { - pending_resps := (pending_resps & - ~UIntToOH(io.data.resp.bits.addr_beat)) | - UIntToOH(curr_read_beat) } } is(s_data_resp) { @@ -918,7 +937,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { is(s_data_write) { io.data.write.valid := pending_writes.orR //TODO make sure all acquire data is present when(io.data.write.ready) { - pending_writes := pending_writes & ~UIntToOH(curr_write_beat) when(PopCount(pending_writes) <= UInt(1)) { state := s_inner_grant } } } @@ -950,20 +968,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } } - ignt_q.io.enq.valid := io.inner.acquire.fire() && - (state === s_idle || !xact.hasMultibeatData()) - ignt_q.io.enq.bits.client_xact_id := io.iacq().client_xact_id - ignt_q.io.enq.bits.addr_beat := io.iacq().addr_beat // Handle Get and Put merging - when(io.inner.acquire.fire()) { - val beat = io.iacq().addr_beat - when(io.iacq().hasData()) { - mergeDataPut(beat, io.iacq().wmask(), io.iacq().data) - //iacq_data_valid(beat) := Bool(true) - pending_writes := pending_writes | UIntToOH(io.iacq().addr_beat) - } - when(state != s_idle) { pending_reads := pending_reads | UIntToOH(io.iacq().addr_beat) } + when(io.inner.acquire.fire() && io.iacq().hasData()) { + mergeDataPut(io.iacq().addr_beat, io.iacq().wmask(), io.iacq().data) + when(!xact.hasMultibeatData()) { ignt_q.io.enq.valid := Bool(true) } + //iacq_data_valid(beat) := Bool(true) } assert(!(state != s_idle && io.inner.acquire.fire() && @@ -1007,7 +1017,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_way_en = Reg{ Bits(width = nWays) } val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) } val xact_id = Reg{ UInt() } - val pending_finish = Reg{ io.outer.finish.bits.clone } + val pending_ofin = Reg{ io.outer.finish.bits.clone } val irel_had_data = Reg(init = Bool(false)) val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) @@ -1043,8 +1053,8 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.release.bits.header.src := UInt(bankId) io.outer.grant.ready := Bool(false) // default io.outer.finish.valid := Bool(false) // default - io.outer.finish.bits := pending_finish - val pending_finish_on_ognt = io.ognt().makeFinish() + io.outer.finish.bits := pending_ofin + val pending_ofin_on_ognt = io.ognt().makeFinish() io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) @@ -1133,8 +1143,8 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.grant.ready := Bool(true) when(io.outer.grant.valid) { when(io.ognt().requiresAck()) { - pending_finish.payload := pending_finish_on_ognt - pending_finish.header.dst := io.outer.grant.bits.header.src + pending_ofin.payload := pending_ofin_on_ognt + pending_ofin.header.dst := io.outer.grant.bits.header.src state := s_outer_finish }.otherwise { state := s_wb_resp diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 0d414f0d..16e976ad 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -139,12 +139,19 @@ abstract class XactTracker extends CoherenceAgentModule { val done = Mux(multi, multi_done, inc) (cnt, done) } + def connectOutgoingDataBeatCounter[T <: HasTileLinkData : ClassTag]( in: DecoupledIO[LogicalNetworkIO[T]], beat: UInt = UInt(0)) = { connectDataBeatCounter(in.fire(), in.bits.payload, beat) } + def connectIncomingDataBeatCounter[T <: HasTileLinkData : ClassTag](in: DecoupledIO[LogicalNetworkIO[T]]) = { connectDataBeatCounter(in.fire(), in.bits.payload, UInt(0))._2 } + + def addPendingBit[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[LogicalNetworkIO[T]]) = { + (Fill(in.bits.payload.tlDataBeats, in.fire() && in.bits.payload.hasData()) & + UIntToOH(in.bits.payload.addr_beat)) + } } From 504eedbf893404b3ed3ee1caf7dcec82161d492f Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 17 Mar 2015 18:07:52 -0700 Subject: [PATCH 337/374] fixes in in bit manipulation --- uncore/src/main/scala/cache.scala | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 0680442d..e3783d93 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -464,10 +464,10 @@ abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { Fill(in.bits.refillCycles, in.fire()) & UIntToOH(in.bits.addr_beat) def dropPendingBit[T <: HasL2BeatAddr] (in: DecoupledIO[T]) = - Fill(in.bits.refillCycles, in.fire()) & ~UIntToOH(in.bits.addr_beat) + ~Fill(in.bits.refillCycles, in.fire()) | ~UIntToOH(in.bits.addr_beat) def dropInternalPendingBit[T <: HasL2BeatAddr] (in: ValidIO[T]) = - Fill(in.bits.refillCycles, in.valid) & ~UIntToOH(in.bits.addr_beat) + ~Fill(in.bits.refillCycles, in.valid) | ~UIntToOH(in.bits.addr_beat) } class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTracker { @@ -634,22 +634,22 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val ifin_cnt = Reg(init = UInt(0, width = log2Up(nSecondaryMisses+1))) when(ignt_data_done) { ifin_cnt := ifin_cnt + UInt(1) } val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) - pending_reads := pending_reads | - addPendingBit(io.inner.acquire) | + pending_reads := (pending_reads | + addPendingBit(io.inner.acquire)) & dropPendingBit(io.data.read) val curr_read_beat = PriorityEncoder(pending_reads) val pending_writes = Reg(init=Bits(0, width = innerDataBeats)) - pending_writes := pending_writes | + pending_writes := (pending_writes | addPendingBit(io.inner.acquire) | addPendingBit(io.inner.release) | - addPendingBit(io.outer.grant) & + addPendingBit(io.outer.grant)) & dropPendingBit(io.data.write) val curr_write_beat = PriorityEncoder(pending_writes) val pending_resps = Reg(init=Bits(0, width = innerDataBeats)) - pending_resps := pending_resps | - addInternalPendingBit(io.data.read) & + pending_resps := (pending_resps | + addInternalPendingBit(io.data.read)) & dropInternalPendingBit(io.data.resp) val pending_coh_on_hit = HierarchicalMetadata( From 2d7375760dd6e5c89ef019a6809833ff1157f39a Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 17 Mar 2015 18:35:04 -0700 Subject: [PATCH 338/374] set pending_writes for puts --- uncore/src/main/scala/cache.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index e3783d93..27c2e64f 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -814,7 +814,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_reads := Mux(io.iacq().isSubBlockType(), UIntToOH(io.iacq().addr_beat), SInt(-1, width = innerDataBeats)).toUInt - pending_writes := UInt(0) + pending_writes := addPendingBit(io.inner.acquire) pending_resps := UInt(0) ifin_cnt := UInt(0) ignt_q.io.enq.valid := Bool(true) From d14efce0b4a6eaea52bc6d83f6cd89839a547e53 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 17 Mar 2015 19:54:11 -0700 Subject: [PATCH 339/374] fix wmask_buffer initialization --- uncore/src/main/scala/cache.scala | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 27c2e64f..9dc35500 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -676,12 +676,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { amoalu.io.lhs := io.data.resp.bits.data //default amoalu.io.rhs := data_buffer.head // default - def mergeDataPut(beat: UInt, wmask: UInt, put_data: UInt) { - val full = FillInterleaved(8, wmask) - data_buffer(beat) := (~full & data_buffer(beat)) | (full & put_data) - wmask_buffer(beat) := wmask | wmask_buffer(beat) - } - def mergeData(dataBits: Int)(beat: UInt, incoming: UInt) { val old_data = incoming // Refilled, written back, or de-cached data val new_data = data_buffer(beat) // Newly Put data is already in the buffer @@ -971,7 +965,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { // Handle Get and Put merging when(io.inner.acquire.fire() && io.iacq().hasData()) { - mergeDataPut(io.iacq().addr_beat, io.iacq().wmask(), io.iacq().data) + val beat = io.iacq().addr_beat + val wmask = io.iacq().wmask() + val full = FillInterleaved(8, wmask) + data_buffer(beat) := (~full & data_buffer(beat)) | (full & io.iacq().data) + wmask_buffer(beat) := wmask | Mux(state === s_idle, Bits(0), wmask_buffer(beat)) when(!xact.hasMultibeatData()) { ignt_q.io.enq.valid := Bool(true) } //iacq_data_valid(beat) := Bool(true) } From 9de5161d7ab8d351a3e86cbed7d8ad7faaac7b25 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 17 Mar 2015 20:24:04 -0700 Subject: [PATCH 340/374] guard all writes to data ram with masks --- uncore/src/main/scala/cache.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 9dc35500..7e14c8f4 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -686,6 +686,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { wmask & Mux(xact.isBuiltInType(Acquire.putAtomicType), amoalu.io.out << xact.amo_shift_bits(), new_data) + wmask_buffer(beat) := SInt(-1) when(xact.is(Acquire.putAtomicType) && xact.addr_beat === beat) { amo_result := old_data } } val mergeDataInternal = mergeData(rowBits) _ @@ -779,7 +780,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.data.write.bits.way_en := xact_way_en io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) io.data.write.bits.addr_beat := curr_write_beat - io.data.write.bits.wmask := SInt(-1) + io.data.write.bits.wmask := wmask_buffer(curr_write_beat) io.data.write.bits.data := data_buffer(curr_write_beat) io.meta.read.valid := Bool(false) io.meta.read.bits.id := UInt(trackerId) From 2d3f947a9c7247317aa9db5b4be30b9e2abe6e86 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 19:33:05 -0700 Subject: [PATCH 341/374] cleaned up finish counter --- uncore/src/main/scala/cache.scala | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 7e14c8f4..c7d857d3 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -632,7 +632,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { ignt_q.io.deq.ready := ignt_data_done val ifin_cnt = Reg(init = UInt(0, width = log2Up(nSecondaryMisses+1))) - when(ignt_data_done) { ifin_cnt := ifin_cnt + UInt(1) } + when(ignt_data_done) { ifin_cnt := ifin_cnt + Mux(io.inner.finish.fire(), UInt(0), UInt(1)) } + .elsewhen(io.inner.finish.fire()) { ifin_cnt := ifin_cnt - UInt(1) } + val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) pending_reads := (pending_reads | addPendingBit(io.inner.acquire)) & @@ -943,12 +945,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { Mux(io.ignt().requiresAck(), s_inner_finish, s_idle)) } io.inner.finish.ready := Bool(true) - when(io.inner.finish.valid) { - ifin_cnt := ifin_cnt - Mux(ignt_data_done, UInt(0), UInt(1)) - } } is(s_meta_write) { io.meta.write.valid := Bool(true) + io.inner.finish.ready := Bool(true) when(io.meta.write.ready) { state := Mux(io.ignt().requiresAck(), s_inner_finish, s_idle) } @@ -956,14 +956,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { is(s_inner_finish) { io.inner.finish.ready := Bool(true) when(io.inner.finish.valid) { - ifin_cnt := ifin_cnt - UInt(1) when(ifin_cnt <= UInt(1)) { state := s_idle } } when(ifin_cnt === UInt(0)) { state := s_idle } } } - // Handle Get and Put merging when(io.inner.acquire.fire() && io.iacq().hasData()) { val beat = io.iacq().addr_beat From b9591b297c94106de1b5bf4560a68a7b95e7751b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 19:59:22 -0700 Subject: [PATCH 342/374] added s_wait_puts to L2AcquireTracker --- uncore/src/main/scala/cache.scala | 33 ++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index c7d857d3..70411a14 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -591,7 +591,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val io = new L2XactTrackerIO - val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_probe :: s_outer_acquire :: s_outer_grant :: s_outer_finish :: s_data_read :: s_data_resp :: s_data_write :: s_inner_grant :: s_meta_write :: s_inner_finish :: Nil = Enum(UInt(), 15) + val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_probe :: s_outer_acquire :: s_outer_grant :: s_outer_finish :: s_data_read :: s_data_resp :: s_wait_puts :: s_data_write :: s_inner_grant :: s_meta_write :: s_inner_finish :: Nil = Enum(UInt(), 16) val state = Reg(init=s_idle) val xact_src = Reg(io.inner.acquire.bits.header.src.clone) @@ -602,12 +602,15 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } val pending_coh = Reg{ xact_meta.coh.clone } + val pending_puts = Reg(init=Bits(0, width = innerDataBeats)) + pending_puts := (pending_puts | addPendingBit(io.inner.acquire)) val is_hit = xact_tag_match && xact_meta.coh.outer.isHit(xact.op_code()) val do_allocate = xact.allocate() val needs_writeback = !xact_tag_match && do_allocate && (xact_meta.coh.outer.requiresVoluntaryWriteback() || xact_meta.coh.inner.requiresProbesOnVoluntaryWriteback()) + val needs_more_put_data = !pending_puts.andR val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) @@ -808,9 +811,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { xact := io.iacq() xact.data := UInt(0) wmask_buffer.foreach { w => w := UInt(0) } - pending_reads := Mux(io.iacq().isSubBlockType(), - UIntToOH(io.iacq().addr_beat), - SInt(-1, width = innerDataBeats)).toUInt + pending_puts := Mux(io.iacq().isBuiltInType(Acquire.putBlockType), + UInt(0), + SInt(-1, width = innerDataBeats)).toUInt + pending_reads := Mux(io.iacq().isSubBlockType(), + UIntToOH(io.iacq().addr_beat), + SInt(-1, width = innerDataBeats)).toUInt pending_writes := addPendingBit(io.inner.acquire) pending_resps := UInt(0) ifin_cnt := UInt(0) @@ -876,7 +882,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } when(release_count === UInt(0)) { state := Mux(is_hit, - Mux(pending_writes.orR, s_data_write, s_data_read), + Mux(pending_writes.orR, + Mux(needs_more_put_data, s_wait_puts, s_data_write), + s_data_read), s_outer_acquire) } } @@ -901,7 +909,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { state := s_outer_finish }.otherwise { state := Mux(!do_allocate, s_inner_grant, - Mux(pending_writes.orR, s_data_write, s_data_read)) + Mux(pending_writes.orR, + Mux(needs_more_put_data, s_wait_puts, s_data_write), + s_data_read)) } } } @@ -910,7 +920,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.finish.valid := Bool(true) when(io.outer.finish.ready) { state := Mux(!do_allocate, s_inner_grant, - Mux(pending_writes.orR, s_data_write, s_data_read)) + Mux(pending_writes.orR, + Mux(needs_more_put_data, s_wait_puts, s_data_write), + s_data_read)) } } is(s_data_read) { @@ -927,12 +939,15 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) pending_resps := pending_resps & ~UIntToOH(io.data.resp.bits.addr_beat) when(PopCount(pending_resps) <= UInt(1)) { - state := Mux(pending_writes.orR, s_data_write, s_inner_grant) + state := Mux(pending_writes.orR, + Mux(needs_more_put_data, s_wait_puts, s_data_write), + s_inner_grant) } } } + is(s_wait_puts) { when(!needs_more_put_data) { state := s_data_write } } is(s_data_write) { - io.data.write.valid := pending_writes.orR //TODO make sure all acquire data is present + io.data.write.valid := pending_writes.orR when(io.data.write.ready) { when(PopCount(pending_writes) <= UInt(1)) { state := s_inner_grant } } From 638bace858b78f7a8bae7bccf6398ead12203d5f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 20:28:06 -0700 Subject: [PATCH 343/374] avoid reading data when write mask is full --- uncore/src/main/scala/cache.scala | 19 ++++++++++++------- uncore/src/main/scala/uncore.scala | 7 ++++++- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 70411a14..b193fc8a 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -603,7 +603,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_way_en = Reg{ Bits(width = nWays) } val pending_coh = Reg{ xact_meta.coh.clone } val pending_puts = Reg(init=Bits(0, width = innerDataBeats)) - pending_puts := (pending_puts | addPendingBit(io.inner.acquire)) + pending_puts := (pending_puts | addPendingBitWhenHasData(io.inner.acquire)) val is_hit = xact_tag_match && xact_meta.coh.outer.isHit(xact.op_code()) val do_allocate = xact.allocate() @@ -640,15 +640,15 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) pending_reads := (pending_reads | - addPendingBit(io.inner.acquire)) & + addPendingBitWhenWmaskIsNotFull(io.inner.acquire)) & dropPendingBit(io.data.read) val curr_read_beat = PriorityEncoder(pending_reads) val pending_writes = Reg(init=Bits(0, width = innerDataBeats)) pending_writes := (pending_writes | - addPendingBit(io.inner.acquire) | - addPendingBit(io.inner.release) | - addPendingBit(io.outer.grant)) & + addPendingBitWhenHasData(io.inner.acquire) | + addPendingBitWhenHasData(io.inner.release) | + addPendingBitWhenHasData(io.outer.grant)) & dropPendingBit(io.data.write) val curr_write_beat = PriorityEncoder(pending_writes) @@ -815,9 +815,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { UInt(0), SInt(-1, width = innerDataBeats)).toUInt pending_reads := Mux(io.iacq().isSubBlockType(), - UIntToOH(io.iacq().addr_beat), + addPendingBitWhenWmaskIsNotFull(io.inner.acquire), SInt(-1, width = innerDataBeats)).toUInt - pending_writes := addPendingBit(io.inner.acquire) + pending_writes := addPendingBitWhenHasData(io.inner.acquire) pending_resps := UInt(0) ifin_cnt := UInt(0) ignt_q.io.enq.valid := Bool(true) @@ -933,6 +933,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.data.resp.valid) { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) } + when(PopCount(pending_reads) === UInt(0)) { + state := Mux(pending_writes.orR, + Mux(needs_more_put_data, s_wait_puts, s_data_write), + s_inner_grant) + } } is(s_data_resp) { when(io.data.resp.valid) { diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 16e976ad..14414b76 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -150,8 +150,13 @@ abstract class XactTracker extends CoherenceAgentModule { connectDataBeatCounter(in.fire(), in.bits.payload, UInt(0))._2 } - def addPendingBit[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[LogicalNetworkIO[T]]) = { + def addPendingBitWhenHasData[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[LogicalNetworkIO[T]]) = { (Fill(in.bits.payload.tlDataBeats, in.fire() && in.bits.payload.hasData()) & UIntToOH(in.bits.payload.addr_beat)) } + + def addPendingBitWhenWmaskIsNotFull(in: DecoupledIO[LogicalNetworkIO[Acquire]]) = { + (Fill(in.bits.payload.tlDataBeats, in.fire() && !in.bits.payload.wmask().andR) & + UIntToOH(in.bits.payload.addr_beat)) + } } From d48775eecb3c7bc36b10959e9fdda6c8562483a1 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 20:31:23 -0700 Subject: [PATCH 344/374] cleanup outdated comments --- uncore/src/main/scala/cache.scala | 9 --------- 1 file changed, 9 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index b193fc8a..8bcf3885 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -869,8 +869,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.inner.release.valid) { pending_coh.inner := pending_icoh_on_irel // Handle released dirty data - //TODO: make sure cacq data is actually present before accpeting - // release data to merge! when(io.irel().hasData()) { pending_coh.outer := pending_ocoh_on_irel mergeDataInner(io.irel().addr_beat, io.irel().data) @@ -990,18 +988,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { data_buffer(beat) := (~full & data_buffer(beat)) | (full & io.iacq().data) wmask_buffer(beat) := wmask | Mux(state === s_idle, Bits(0), wmask_buffer(beat)) when(!xact.hasMultibeatData()) { ignt_q.io.enq.valid := Bool(true) } - //iacq_data_valid(beat) := Bool(true) } assert(!(state != s_idle && io.inner.acquire.fire() && io.inner.acquire.bits.header.src != xact_src), "AcquireTracker accepted data beat from different network source than initial request.") - - //TODO: Assumes in-order network - assert(!(state === s_idle && io.inner.acquire.fire() && - !io.iacq().isSubBlockType() && - io.iacq().addr_beat != UInt(0)), - "AcquireTracker initialized with a tail data beat.") } class L2WritebackReq extends L2HellaCacheBundle From 0e4cf74d8ab6c3fedebc0afae80404bf9978267b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 20:53:27 -0700 Subject: [PATCH 345/374] always merge Puts --- uncore/src/main/scala/cache.scala | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 8bcf3885..455f85ef 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -173,7 +173,6 @@ abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgen require(rowBits == innerDataBits) // TODO: relax this by improving s_data_* states val nSecondaryMisses = 4 val enableGetMerging = false - val enablePutMerging = true } abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters @@ -709,11 +708,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { do_allocate && ignt_q.io.enq.ready //TODO: mix Puts and PutBlocks - val can_merge_iacq_put = ((Bool(enablePutMerging) && - (xact.isBuiltInType(Acquire.putType) && - io.iacq().isBuiltInType(Acquire.putType))) || - (xact.isBuiltInType(Acquire.putBlockType) && - io.iacq().isBuiltInType(Acquire.putBlockType))) && + val can_merge_iacq_put = ((xact.isBuiltInType(Acquire.putType) && + io.iacq().isBuiltInType(Acquire.putType)) || + (xact.isBuiltInType(Acquire.putBlockType) && + io.iacq().isBuiltInType(Acquire.putBlockType))) && (xact_src === io.inner.acquire.bits.header.src) && (xact.client_xact_id === io.iacq().client_xact_id) && xact.conflicts(io.iacq()) && From f4f59464df9bce2c8ccec3a9b7a9a698fa53ae7f Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 17 Mar 2015 21:44:20 -0700 Subject: [PATCH 346/374] fix pending_puts initialization --- uncore/src/main/scala/cache.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 455f85ef..514878e4 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -810,7 +810,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { xact.data := UInt(0) wmask_buffer.foreach { w => w := UInt(0) } pending_puts := Mux(io.iacq().isBuiltInType(Acquire.putBlockType), - UInt(0), + addPendingBitWhenHasData(io.inner.acquire), SInt(-1, width = innerDataBeats)).toUInt pending_reads := Mux(io.iacq().isSubBlockType(), addPendingBitWhenWmaskIsNotFull(io.inner.acquire), From aa5435800d7e7f757752817229fc8fd956d1c1f3 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 17 Mar 2015 22:43:00 -0700 Subject: [PATCH 347/374] fix get merging, and always turn it on --- uncore/src/main/scala/cache.scala | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 514878e4..27bb898b 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -172,7 +172,6 @@ abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgen require(amoAluOperandBits <= innerDataBits) require(rowBits == innerDataBits) // TODO: relax this by improving s_data_* states val nSecondaryMisses = 4 - val enableGetMerging = false } abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters @@ -697,8 +696,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val mergeDataInner = mergeData(innerDataBits) _ val mergeDataOuter = mergeData(outerDataBits) _ - val can_merge_iacq_get = Bool(enableGetMerging) && - (xact.isBuiltInType(Acquire.getType) && + val can_merge_iacq_get = (xact.isBuiltInType(Acquire.getType) && io.iacq().isBuiltInType(Acquire.getType)) && (xact_src === io.inner.acquire.bits.header.src) && xact.conflicts(io.iacq()) && @@ -979,13 +977,15 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } // Handle Get and Put merging - when(io.inner.acquire.fire() && io.iacq().hasData()) { - val beat = io.iacq().addr_beat - val wmask = io.iacq().wmask() - val full = FillInterleaved(8, wmask) - data_buffer(beat) := (~full & data_buffer(beat)) | (full & io.iacq().data) - wmask_buffer(beat) := wmask | Mux(state === s_idle, Bits(0), wmask_buffer(beat)) - when(!xact.hasMultibeatData()) { ignt_q.io.enq.valid := Bool(true) } + when(io.inner.acquire.fire()) { + when (io.iacq().hasData()) { + val beat = io.iacq().addr_beat + val wmask = io.iacq().wmask() + val full = FillInterleaved(8, wmask) + data_buffer(beat) := (~full & data_buffer(beat)) | (full & io.iacq().data) + wmask_buffer(beat) := wmask | Mux(state === s_idle, Bits(0), wmask_buffer(beat)) + } + when(!io.iacq().hasMultibeatData()) { ignt_q.io.enq.valid := Bool(true) } } assert(!(state != s_idle && io.inner.acquire.fire() && From 825c4b28509a81ad1d17e0df80c56d5384130f80 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 22:44:53 -0700 Subject: [PATCH 348/374] make ignts more eager --- uncore/src/main/scala/cache.scala | 52 ++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 455f85ef..39b5e7c2 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -459,13 +459,16 @@ abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { connectDataBeatCounter(in.valid, in.bits, UInt(0), full_block)._2 } - def addInternalPendingBit[T <: HasL2BeatAddr](in: DecoupledIO[T]) = + def addPendingBitInternal[T <: HasL2BeatAddr](in: DecoupledIO[T]) = Fill(in.bits.refillCycles, in.fire()) & UIntToOH(in.bits.addr_beat) + def addPendingBitInternal[T <: HasL2BeatAddr](in: ValidIO[T]) = + Fill(in.bits.refillCycles, in.valid) & UIntToOH(in.bits.addr_beat) + def dropPendingBit[T <: HasL2BeatAddr] (in: DecoupledIO[T]) = ~Fill(in.bits.refillCycles, in.fire()) | ~UIntToOH(in.bits.addr_beat) - def dropInternalPendingBit[T <: HasL2BeatAddr] (in: ValidIO[T]) = + def dropPendingBitInternal[T <: HasL2BeatAddr] (in: ValidIO[T]) = ~Fill(in.bits.refillCycles, in.valid) | ~UIntToOH(in.bits.addr_beat) } @@ -626,6 +629,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) val pending_ofin = Reg{ io.outer.finish.bits.clone } + val pending_ignt_data = Reg(init=Bits(0, width = innerDataBeats)) + pending_ignt_data := pending_ignt_data | + addPendingBitInternal(io.data.resp) | + addPendingBitWhenHasData(io.inner.release) | + addPendingBitWhenHasData(io.outer.grant) val ignt_q = Module(new Queue(new L2SecondaryMissInfo, nSecondaryMisses))(innerTLParams) val (ignt_data_idx, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) ignt_q.io.enq.valid := Bool(false) @@ -634,8 +642,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { ignt_q.io.deq.ready := ignt_data_done val ifin_cnt = Reg(init = UInt(0, width = log2Up(nSecondaryMisses+1))) - when(ignt_data_done) { ifin_cnt := ifin_cnt + Mux(io.inner.finish.fire(), UInt(0), UInt(1)) } - .elsewhen(io.inner.finish.fire()) { ifin_cnt := ifin_cnt - UInt(1) } + when(ignt_data_done) { + ifin_cnt := Mux(io.inner.finish.fire(), + Mux(io.ignt().requiresAck(), ifin_cnt, ifin_cnt - UInt(1)), + Mux(io.ignt().requiresAck(), ifin_cnt + UInt(1), ifin_cnt)) + } .elsewhen(io.inner.finish.fire()) { ifin_cnt := ifin_cnt - UInt(1) } val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) pending_reads := (pending_reads | @@ -653,8 +664,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val pending_resps = Reg(init=Bits(0, width = innerDataBeats)) pending_resps := (pending_resps | - addInternalPendingBit(io.data.read)) & - dropInternalPendingBit(io.data.resp) + addPendingBitInternal(io.data.read)) & + dropPendingBitInternal(io.data.resp) val pending_coh_on_hit = HierarchicalMetadata( io.meta.resp.bits.meta.coh.inner, @@ -755,7 +766,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.inner.probe.bits.header.dst := curr_probe_dst io.inner.probe.bits.payload := pending_coh.inner.makeProbe(xact) - io.inner.grant.valid := state === s_inner_grant && ignt_q.io.deq.valid + io.inner.grant.valid := state === s_inner_grant && + ignt_q.io.deq.valid && + (!io.ignt().hasData() || + pending_ignt_data(ignt_data_idx)) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := xact_src io.inner.grant.bits.payload := pending_coh.inner.makeGrant( @@ -770,8 +784,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.inner.acquire.ready := state === s_idle || can_merge_iacq_put || can_merge_iacq_get - io.inner.release.ready := Bool(false) - io.inner.finish.ready := Bool(false) + io.inner.release.ready := state === s_probe + io.inner.finish.ready := Vec(s_inner_finish, s_meta_write, s_inner_grant, + s_data_write, s_wait_puts, s_data_resp).contains(state) io.data.read.valid := Bool(false) io.data.read.bits.id := UInt(trackerId) @@ -817,6 +832,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { SInt(-1, width = innerDataBeats)).toUInt pending_writes := addPendingBitWhenHasData(io.inner.acquire) pending_resps := UInt(0) + pending_ignt_data := UInt(0) ifin_cnt := UInt(0) ignt_q.io.enq.valid := Bool(true) state := s_meta_read @@ -863,7 +879,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_probes := pending_probes & ~UIntToOH(curr_probe_dst) } // Handle releases, which may have data being written back - io.inner.release.ready := Bool(true) when(io.inner.release.valid) { pending_coh.inner := pending_icoh_on_irel // Handle released dirty data @@ -954,27 +969,26 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } } is(s_inner_grant) { - when(ignt_data_done && ignt_q.io.count === UInt(1)) { + when(ignt_q.io.count === UInt(0) || + (ignt_data_done && ignt_q.io.count === UInt(1))) { val meta_dirty = !is_hit || pending_coh_on_ignt != xact_meta.coh when(meta_dirty) { pending_coh := pending_coh_on_ignt } state := Mux(meta_dirty, s_meta_write, - Mux(io.ignt().requiresAck(), s_inner_finish, s_idle)) + Mux(ifin_cnt > UInt(0) || io.ignt().requiresAck(), + s_inner_finish, s_idle)) } - io.inner.finish.ready := Bool(true) } is(s_meta_write) { io.meta.write.valid := Bool(true) - io.inner.finish.ready := Bool(true) when(io.meta.write.ready) { - state := Mux(io.ignt().requiresAck(), s_inner_finish, s_idle) + state := Mux(ifin_cnt > UInt(0), s_inner_finish, s_idle) } } is(s_inner_finish) { - io.inner.finish.ready := Bool(true) - when(io.inner.finish.valid) { - when(ifin_cnt <= UInt(1)) { state := s_idle } + when(ifin_cnt === UInt(0) || + (io.inner.finish.valid && ifin_cnt === UInt(1))) { + state := s_idle } - when(ifin_cnt === UInt(0)) { state := s_idle } } } From 42aa4aa8cab91fad1a5220108f16a47e63483f61 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 22:53:50 -0700 Subject: [PATCH 349/374] Secondary miss param --- uncore/src/main/scala/cache.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index b632bbaa..98e122d4 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -10,6 +10,8 @@ case object RowBits extends Field[Int] case object Replacer extends Field[() => ReplacementPolicy] case object AmoAluOperandBits extends Field[Int] case object L2DirectoryRepresentation extends Field[DirectoryRepresentation] +case object NPrimaryMisses extends Field[Int] +case object NSecondaryMisses extends Field[Int] case object CacheBlockBytes extends Field[Int] case object CacheBlockOffsetBits extends Field[Int] @@ -171,7 +173,7 @@ abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgen val amoAluOperandBits = params(AmoAluOperandBits) require(amoAluOperandBits <= innerDataBits) require(rowBits == innerDataBits) // TODO: relax this by improving s_data_* states - val nSecondaryMisses = 4 + val nSecondaryMisses = params(NSecondaryMisses) } abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters From e325399c8781181b9d030d7bb4f8d85c87d0ea54 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 18 Mar 2015 12:49:53 -0700 Subject: [PATCH 350/374] Re-split mem resp tag and data queues --- uncore/src/main/scala/memserdes.scala | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index a5bea3a3..e2d6df85 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -544,11 +544,21 @@ class MemPipeIOMemIOConverter(numRequests: Int, refillCycles: Int) extends Modul io.mem.req_data <> io.cpu.req_data // Have separate queues to allow for different mem implementations - val resp_data_q = Module((new HellaQueue(numEntries)) { new MemResp }) - resp_data_q.io.enq <> io.mem.resp - io.cpu.resp <> resp_data_q.io.deq + val resp_data_q = Module((new HellaQueue(numEntries)) { new MemData }) + resp_data_q.io.enq.valid := io.mem.resp.valid + resp_data_q.io.enq.bits.data := io.mem.resp.bits.data - inc := resp_data_q.io.deq.fire() + val resp_tag_q = Module((new HellaQueue(numEntries)) { new MemTag }) + resp_tag_q.io.enq.valid := io.mem.resp.valid + resp_tag_q.io.enq.bits.tag := io.mem.resp.bits.tag + + io.cpu.resp.valid := resp_data_q.io.deq.valid && resp_tag_q.io.deq.valid + io.cpu.resp.bits.data := resp_data_q.io.deq.bits.data + io.cpu.resp.bits.tag := resp_tag_q.io.deq.bits.tag + resp_data_q.io.deq.ready := io.cpu.resp.ready + resp_tag_q.io.deq.ready := io.cpu.resp.ready + + inc := resp_data_q.io.deq.fire() && resp_tag_q.io.deq.fire() dec := io.mem.req_cmd.fire() && !io.mem.req_cmd.bits.rw } From 1ff184bf62ea41370a0ee083ee69257dae3dd86f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 18 Mar 2015 17:55:05 -0700 Subject: [PATCH 351/374] first cut at optimized state transitions --- uncore/src/main/scala/cache.scala | 169 ++++++++++++++--------------- uncore/src/main/scala/uncore.scala | 19 +++- 2 files changed, 94 insertions(+), 94 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 98e122d4..85fedbbc 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -174,6 +174,7 @@ abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgen require(amoAluOperandBits <= innerDataBits) require(rowBits == innerDataBits) // TODO: relax this by improving s_data_* states val nSecondaryMisses = params(NSecondaryMisses) + val isLastLevelCache = true } abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters @@ -594,7 +595,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val io = new L2XactTrackerIO - val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_probe :: s_outer_acquire :: s_outer_grant :: s_outer_finish :: s_data_read :: s_data_resp :: s_wait_puts :: s_data_write :: s_inner_grant :: s_meta_write :: s_inner_finish :: Nil = Enum(UInt(), 16) + val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_inner_probe :: s_outer_acquire :: s_outer_grant :: s_outer_finish :: s_data_read :: s_data_resp :: s_wait_puts :: s_data_write :: s_inner_grant :: s_meta_write :: s_inner_finish :: Nil = Enum(UInt(), 16) val state = Reg(init=s_idle) val xact_src = Reg(io.inner.acquire.bits.header.src.clone) @@ -605,15 +606,18 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } val pending_coh = Reg{ xact_meta.coh.clone } - val pending_puts = Reg(init=Bits(0, width = innerDataBeats)) - pending_puts := (pending_puts | addPendingBitWhenHasData(io.inner.acquire)) + val present_puts = Reg(init=Bits(0, width = innerDataBeats)) + present_puts := (present_puts | addPendingBitWhenHasData(io.inner.acquire)) - val is_hit = xact_tag_match && xact_meta.coh.outer.isHit(xact.op_code()) + val is_hit = xact_tag_match && + (xact_meta.coh.outer.isHit(xact.op_code()) || + (Bool(isLastLevelCache) && // LLC has all the permissions + xact_meta.coh.outer.isValid())) val do_allocate = xact.allocate() val needs_writeback = !xact_tag_match && do_allocate && (xact_meta.coh.outer.requiresVoluntaryWriteback() || xact_meta.coh.inner.requiresProbesOnVoluntaryWriteback()) - val needs_more_put_data = !pending_puts.andR + val needs_more_put_data = !present_puts.andR val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) @@ -652,7 +656,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) pending_reads := (pending_reads | addPendingBitWhenWmaskIsNotFull(io.inner.acquire)) & - dropPendingBit(io.data.read) + (dropPendingBit(io.data.read) & + dropPendingBitWhenWmaskIsFull(io.inner.acquire) & + dropPendingBitWhenHasData(io.inner.release) & + dropPendingBitWhenHasData(io.outer.grant)) val curr_read_beat = PriorityEncoder(pending_reads) val pending_writes = Reg(init=Bits(0, width = innerDataBeats)) @@ -683,6 +690,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { outgoing = io.ignt(), dst = io.inner.grant.bits.header.dst), pending_coh.outer) + val pending_ofin_on_ognt = io.ognt().makeFinish() val amo_result = xact.data val amoalu = Module(new AMOALU) @@ -714,11 +722,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { (xact_src === io.inner.acquire.bits.header.src) && xact.conflicts(io.iacq()) && Vec(s_meta_read, s_meta_resp, s_wb_req, s_wb_resp, - s_probe, s_outer_acquire, s_outer_grant, + s_inner_probe, s_outer_acquire, s_outer_grant, s_outer_finish).contains(state) && do_allocate && ignt_q.io.enq.ready - //TODO: mix Puts and PutBlocks + val can_merge_iacq_put = ((xact.isBuiltInType(Acquire.putType) && io.iacq().isBuiltInType(Acquire.putType)) || (xact.isBuiltInType(Acquire.putBlockType) && @@ -727,7 +735,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { (xact.client_xact_id === io.iacq().client_xact_id) && xact.conflicts(io.iacq()) && Vec(s_meta_read, s_meta_resp, s_wb_req, s_wb_resp, - s_probe, s_outer_acquire, s_outer_grant, + s_inner_probe, s_outer_acquire, s_outer_grant, s_outer_finish, s_data_read, s_data_resp).contains(state) && do_allocate && @@ -737,7 +745,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.iacq().addr_block(idxMSB,idxLSB) io.has_release_match := xact.conflicts(io.irel()) && !io.irel().isVoluntary() && - (state === s_probe) + (state === s_inner_probe) io.has_acquire_match := can_merge_iacq_put || can_merge_iacq_get io.has_acquire_conflict := (xact.conflicts(io.iacq()) || in_same_set) && (state != s_idle) && @@ -746,7 +754,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { // If we're allocating in this cache, we can use the current metadata // to make an appropriate custom Acquire, otherwise we copy over the // built-in Acquire from the inner TL to the outer TL - io.outer.acquire.valid := Bool(false) + io.outer.acquire.valid := state === s_outer_acquire io.outer.acquire.bits.payload := Mux(do_allocate, xact_meta.coh.outer.makeAcquire( client_xact_id = UInt(trackerId), @@ -756,12 +764,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.acquire.bits.header.src := UInt(bankId) io.outer.probe.ready := Bool(false) io.outer.release.valid := Bool(false) - io.outer.grant.ready := Bool(false) - io.outer.finish.valid := Bool(false) + io.outer.grant.ready := state === s_outer_grant + io.outer.finish.valid := state === s_outer_finish io.outer.finish.bits := pending_ofin - val pending_ofin_on_ognt = io.ognt().makeFinish() - io.inner.probe.valid := Bool(false) + io.inner.probe.valid := state === s_inner_probe && pending_probes.orR io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_probe_dst io.inner.probe.bits.payload := pending_coh.inner.makeProbe(xact) @@ -784,34 +791,34 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.inner.acquire.ready := state === s_idle || can_merge_iacq_put || can_merge_iacq_get - io.inner.release.ready := state === s_probe + io.inner.release.ready := state === s_inner_probe io.inner.finish.ready := Vec(s_inner_finish, s_meta_write, s_inner_grant, s_data_write, s_wait_puts, s_data_resp).contains(state) - io.data.read.valid := Bool(false) + io.data.read.valid := state === s_data_read && pending_reads.orR io.data.read.bits.id := UInt(trackerId) io.data.read.bits.way_en := xact_way_en io.data.read.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) io.data.read.bits.addr_beat := curr_read_beat - io.data.write.valid := Bool(false) + io.data.write.valid := state === s_data_write && pending_writes.orR io.data.write.bits.id := UInt(trackerId) io.data.write.bits.way_en := xact_way_en io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) io.data.write.bits.addr_beat := curr_write_beat io.data.write.bits.wmask := wmask_buffer(curr_write_beat) io.data.write.bits.data := data_buffer(curr_write_beat) - io.meta.read.valid := Bool(false) + io.meta.read.valid := state === s_meta_read io.meta.read.bits.id := UInt(trackerId) io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) io.meta.read.bits.tag := xact.addr_block >> UInt(idxBits) - io.meta.write.valid := Bool(false) + io.meta.write.valid := state === s_meta_write io.meta.write.bits.id := UInt(trackerId) io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en io.meta.write.bits.data.tag := xact.addr_block >> UInt(idxBits) io.meta.write.bits.data.coh := pending_coh - io.wb.req.valid := Bool(false) + io.wb.req.valid := state === s_wb_req io.wb.req.bits.addr_block := Cat(xact_meta.tag, xact.addr_block(idxMSB,idxLSB)) io.wb.req.bits.coh := xact_meta.coh io.wb.req.bits.way_en := xact_way_en @@ -824,12 +831,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { xact := io.iacq() xact.data := UInt(0) wmask_buffer.foreach { w => w := UInt(0) } - pending_puts := Mux(io.iacq().isBuiltInType(Acquire.putBlockType), + present_puts := Mux(io.iacq().isBuiltInType(Acquire.putBlockType), addPendingBitWhenHasData(io.inner.acquire), SInt(-1, width = innerDataBeats)).toUInt - pending_reads := Mux(io.iacq().isSubBlockType(), - addPendingBitWhenWmaskIsNotFull(io.inner.acquire), - SInt(-1, width = innerDataBeats)).toUInt + pending_reads := Mux(io.iacq().isBuiltInType(Acquire.putBlockType), + UInt(0), + Mux(io.iacq().isSubBlockType(), + addPendingBitWhenWmaskIsNotFull(io.inner.acquire), + SInt(-1, width = innerDataBeats)).toUInt) pending_writes := addPendingBitWhenHasData(io.inner.acquire) pending_resps := UInt(0) pending_ignt_data := UInt(0) @@ -838,10 +847,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { state := s_meta_read } } - is(s_meta_read) { - io.meta.read.valid := Bool(true) - when(io.meta.read.ready) { state := s_meta_resp } - } + is(s_meta_read) { when(io.meta.read.ready) { state := s_meta_resp } } is(s_meta_resp) { when(io.meta.resp.valid) { xact_tag_match := io.meta.resp.bits.tag_match @@ -850,31 +856,33 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_coh := io.meta.resp.bits.meta.coh val _coh = io.meta.resp.bits.meta.coh val _tag_match = io.meta.resp.bits.tag_match - val _is_hit = _tag_match && _coh.outer.isHit(xact.op_code()) + val _is_hit = _tag_match && + (_coh.outer.isHit(xact.op_code()) || + (Bool(isLastLevelCache) && // LLC has all the permissions + _coh.outer.isValid())) + val _needs_writeback = !_tag_match && do_allocate && (_coh.outer.requiresVoluntaryWriteback() || _coh.inner.requiresProbesOnVoluntaryWriteback()) - val _needs_probes = _tag_match && _coh.inner.requiresProbes(xact) + val _needs_inner_probes = _tag_match && _coh.inner.requiresProbes(xact) when(_is_hit) { pending_coh := pending_coh_on_hit } - when(_needs_probes) { + when(_needs_inner_probes) { pending_probes := mask_incoherent release_count := PopCount(mask_incoherent) } - state := Mux(_tag_match, - Mux(_needs_probes, s_probe, Mux(_is_hit, s_data_read, s_outer_acquire)), // Probe, hit or upgrade - Mux(_needs_writeback, s_wb_req, s_outer_acquire)) // Evict ifneedbe + state := Mux(!_tag_match, + Mux(_needs_writeback, s_wb_req, s_outer_acquire), + Mux(_needs_inner_probes, s_inner_probe, + Mux(!is_hit, s_outer_acquire, + Mux(pending_reads.orR, s_data_read, + Mux(!pending_writes.orR, s_inner_grant, + Mux(needs_more_put_data, s_wait_puts, s_data_write)))))) } } - is(s_wb_req) { - io.wb.req.valid := Bool(true) - when(io.wb.req.ready) { state := s_wb_resp } - } - is(s_wb_resp) { - when(io.wb.resp.valid) { state := s_outer_acquire } - } - is(s_probe) { + is(s_wb_req) { when(io.wb.req.ready) { state := s_wb_resp } } + is(s_wb_resp) { when(io.wb.resp.valid) { state := s_outer_acquire } } + is(s_inner_probe) { // Send probes - io.inner.probe.valid := pending_probes != UInt(0) when(io.inner.probe.ready) { pending_probes := pending_probes & ~UIntToOH(curr_probe_dst) } @@ -892,21 +900,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } } when(release_count === UInt(0)) { - state := Mux(is_hit, - Mux(pending_writes.orR, - Mux(needs_more_put_data, s_wait_puts, s_data_write), - s_data_read), - s_outer_acquire) - } - } - is(s_outer_acquire) { - io.outer.acquire.valid := Bool(true) - when(oacq_data_done) { - state := s_outer_grant + state := Mux(!is_hit, s_outer_acquire, + Mux(pending_reads.orR, s_data_read, + Mux(!pending_writes.orR, s_inner_grant, + Mux(needs_more_put_data, s_wait_puts, s_data_write)))) } } + is(s_outer_acquire) { when(oacq_data_done) { state := s_outer_grant } } is(s_outer_grant) { - io.outer.grant.ready := Bool(true) when(io.outer.grant.valid) { when(io.ognt().hasData()) { mergeDataOuter(io.ognt().addr_beat, io.ognt().data) @@ -919,51 +920,40 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_ofin.header.src := UInt(bankId) state := s_outer_finish }.otherwise { - state := Mux(!do_allocate, s_inner_grant, - Mux(pending_writes.orR, - Mux(needs_more_put_data, s_wait_puts, s_data_write), - s_data_read)) + state := Mux(pending_reads.orR, s_data_read, + Mux(!pending_writes.orR, s_inner_grant, + Mux(needs_more_put_data, s_wait_puts, s_data_write))) } } } } is(s_outer_finish) { - io.outer.finish.valid := Bool(true) when(io.outer.finish.ready) { - state := Mux(!do_allocate, s_inner_grant, - Mux(pending_writes.orR, - Mux(needs_more_put_data, s_wait_puts, s_data_write), - s_data_read)) + state := Mux(pending_reads.orR, s_data_read, + Mux(!pending_writes.orR, s_inner_grant, + Mux(needs_more_put_data, s_wait_puts, s_data_write))) } } is(s_data_read) { - io.data.read.valid := pending_reads.orR - when(io.data.read.ready) { - when(PopCount(pending_reads) <= UInt(1)) { state := s_data_resp } - } when(io.data.resp.valid) { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) } - when(PopCount(pending_reads) === UInt(0)) { - state := Mux(pending_writes.orR, - Mux(needs_more_put_data, s_wait_puts, s_data_write), - s_inner_grant) + when(io.data.read.ready) { + when(PopCount(pending_reads) <= UInt(1)) { state := s_data_resp } } } is(s_data_resp) { when(io.data.resp.valid) { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) - pending_resps := pending_resps & ~UIntToOH(io.data.resp.bits.addr_beat) - when(PopCount(pending_resps) <= UInt(1)) { - state := Mux(pending_writes.orR, - Mux(needs_more_put_data, s_wait_puts, s_data_write), - s_inner_grant) - } + } + when(PopCount(pending_resps) === UInt(0) || + (io.data.resp.valid && PopCount(pending_resps) === UInt(1))) { + state := Mux(!pending_writes.orR, s_inner_grant, + Mux(needs_more_put_data, s_wait_puts, s_data_write)) } } is(s_wait_puts) { when(!needs_more_put_data) { state := s_data_write } } is(s_data_write) { - io.data.write.valid := pending_writes.orR when(io.data.write.ready) { when(PopCount(pending_writes) <= UInt(1)) { state := s_inner_grant } } @@ -978,15 +968,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { s_inner_finish, s_idle)) } } - is(s_meta_write) { - io.meta.write.valid := Bool(true) + is(s_meta_write) { when(io.meta.write.ready) { state := Mux(ifin_cnt > UInt(0), s_inner_finish, s_idle) } } is(s_inner_finish) { when(ifin_cnt === UInt(0) || - (io.inner.finish.valid && ifin_cnt === UInt(1))) { + (io.inner.finish.valid && ifin_cnt === UInt(1))) { state := s_idle } } @@ -1031,7 +1020,7 @@ class L2WritebackUnitIO extends HierarchicalXactTrackerIO { class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val io = new L2WritebackUnitIO - val s_idle :: s_probe :: s_data_read :: s_data_resp :: s_outer_release :: s_outer_grant :: s_outer_finish :: s_wb_resp :: Nil = Enum(UInt(), 8) + val s_idle :: s_inner_probe :: s_data_read :: s_data_resp :: s_outer_release :: s_outer_grant :: s_outer_finish :: s_wb_resp :: Nil = Enum(UInt(), 8) val state = Reg(init=s_idle) val xact_addr_block = Reg(io.wb.req.bits.addr_block.clone) @@ -1057,12 +1046,13 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { incoming = io.irel(), src = io.inner.release.bits.header.src) val pending_ocoh_on_irel = xact_coh.outer.onHit(M_XWR) // WB is a write + val pending_ofin_on_ognt = io.ognt().makeFinish() io.has_acquire_conflict := Bool(false) io.has_acquire_match := Bool(false) io.has_release_match := !io.irel().isVoluntary() && io.irel().conflicts(xact_addr_block) && - (state === s_probe) + (state === s_inner_probe) io.outer.acquire.valid := Bool(false) io.outer.probe.ready := Bool(false) @@ -1076,7 +1066,6 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.grant.ready := Bool(false) // default io.outer.finish.valid := Bool(false) // default io.outer.finish.bits := pending_ofin - val pending_ofin_on_ognt = io.ognt().makeFinish() io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) @@ -1109,15 +1098,15 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { xact_way_en := io.wb.req.bits.way_en xact_id := io.wb.req.bits.id irel_had_data := Bool(false) - val needs_probes = io.wb.req.bits.coh.inner.requiresProbesOnVoluntaryWriteback() - when(needs_probes) { + val needs_inner_probes = io.wb.req.bits.coh.inner.requiresProbesOnVoluntaryWriteback() + when(needs_inner_probes) { pending_probes := mask_incoherent release_count := PopCount(mask_incoherent) } - state := Mux(needs_probes, s_probe, s_data_read) + state := Mux(needs_inner_probes, s_inner_probe, s_data_read) } } - is(s_probe) { + is(s_inner_probe) { // Send probes io.inner.probe.valid := pending_probes != UInt(0) when(io.inner.probe.ready) { diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 14414b76..cfd33567 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -151,12 +151,23 @@ abstract class XactTracker extends CoherenceAgentModule { } def addPendingBitWhenHasData[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[LogicalNetworkIO[T]]) = { - (Fill(in.bits.payload.tlDataBeats, in.fire() && in.bits.payload.hasData()) & - UIntToOH(in.bits.payload.addr_beat)) + Fill(in.bits.payload.tlDataBeats, in.fire() && in.bits.payload.hasData()) & + UIntToOH(in.bits.payload.addr_beat) } + def dropPendingBitWhenHasData[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[LogicalNetworkIO[T]]) = { + ~Fill(in.bits.payload.tlDataBeats, in.fire() && in.bits.payload.hasData()) | + ~UIntToOH(in.bits.payload.addr_beat) + } + + //TODO | with existing wmask_buffer? def addPendingBitWhenWmaskIsNotFull(in: DecoupledIO[LogicalNetworkIO[Acquire]]) = { - (Fill(in.bits.payload.tlDataBeats, in.fire() && !in.bits.payload.wmask().andR) & - UIntToOH(in.bits.payload.addr_beat)) + Fill(in.bits.payload.tlDataBeats, in.fire() && !in.bits.payload.wmask().andR) & + UIntToOH(in.bits.payload.addr_beat) + } + + def dropPendingBitWhenWmaskIsFull(in: DecoupledIO[LogicalNetworkIO[Acquire]]) = { + ~Fill(in.bits.payload.tlDataBeats, in.fire() && in.bits.payload.wmask().andR) | + ~UIntToOH(in.bits.payload.addr_beat) } } From 19059bf0ebc0e3e2f2c6bfb746ff19a04208e6dd Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 18 Mar 2015 18:28:03 -0700 Subject: [PATCH 352/374] put data can be used for ignts --- uncore/src/main/scala/cache.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 85fedbbc..7257a699 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -638,7 +638,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_ignt_data := pending_ignt_data | addPendingBitInternal(io.data.resp) | addPendingBitWhenHasData(io.inner.release) | - addPendingBitWhenHasData(io.outer.grant) + addPendingBitWhenHasData(io.outer.grant) | + addPendingBitWhenHasData(io.inner.acquire) val ignt_q = Module(new Queue(new L2SecondaryMissInfo, nSecondaryMisses))(innerTLParams) val (ignt_data_idx, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) ignt_q.io.enq.valid := Bool(false) From fb8071c12d90b20803c933f7a5a416855691cc44 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 18 Mar 2015 18:49:32 -0700 Subject: [PATCH 353/374] generous hit detection on PutBlocks --- uncore/src/main/scala/cache.scala | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 7257a699..741e168d 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -609,10 +609,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val present_puts = Reg(init=Bits(0, width = innerDataBeats)) present_puts := (present_puts | addPendingBitWhenHasData(io.inner.acquire)) - val is_hit = xact_tag_match && - (xact_meta.coh.outer.isHit(xact.op_code()) || - (Bool(isLastLevelCache) && // LLC has all the permissions - xact_meta.coh.outer.isValid())) + val is_hit = (if(isLastLevelCache) + (xact.isBuiltInType(Acquire.putBlockType) || + xact_meta.coh.outer.isValid()) + else (xact_tag_match && xact_meta.coh.outer.isHit(xact.op_code()))) val do_allocate = xact.allocate() val needs_writeback = !xact_tag_match && do_allocate && (xact_meta.coh.outer.requiresVoluntaryWriteback() || @@ -857,10 +857,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_coh := io.meta.resp.bits.meta.coh val _coh = io.meta.resp.bits.meta.coh val _tag_match = io.meta.resp.bits.tag_match - val _is_hit = _tag_match && - (_coh.outer.isHit(xact.op_code()) || - (Bool(isLastLevelCache) && // LLC has all the permissions - _coh.outer.isValid())) + val _is_hit = (if(isLastLevelCache) + (xact.isBuiltInType(Acquire.putBlockType) || + _coh.outer.isValid()) + else (_tag_match && _coh.outer.isHit(xact.op_code()))) val _needs_writeback = !_tag_match && do_allocate && (_coh.outer.requiresVoluntaryWriteback() || From b92ea608914be20519f57b3d1189c27cd794c85c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 18 Mar 2015 19:32:46 -0700 Subject: [PATCH 354/374] you can 'hit' with putblocks even when the tag doesn't match but you still better writeback --- uncore/src/main/scala/cache.scala | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 741e168d..146250bc 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -859,7 +859,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val _tag_match = io.meta.resp.bits.tag_match val _is_hit = (if(isLastLevelCache) (xact.isBuiltInType(Acquire.putBlockType) || - _coh.outer.isValid()) + _tag_match && _coh.outer.isValid()) else (_tag_match && _coh.outer.isHit(xact.op_code()))) val _needs_writeback = !_tag_match && do_allocate && @@ -871,17 +871,23 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_probes := mask_incoherent release_count := PopCount(mask_incoherent) } - state := Mux(!_tag_match, - Mux(_needs_writeback, s_wb_req, s_outer_acquire), + state := Mux(_needs_writeback, s_wb_req, Mux(_needs_inner_probes, s_inner_probe, - Mux(!is_hit, s_outer_acquire, + Mux(!_is_hit, s_outer_acquire, Mux(pending_reads.orR, s_data_read, Mux(!pending_writes.orR, s_inner_grant, Mux(needs_more_put_data, s_wait_puts, s_data_write)))))) } } is(s_wb_req) { when(io.wb.req.ready) { state := s_wb_resp } } - is(s_wb_resp) { when(io.wb.resp.valid) { state := s_outer_acquire } } + is(s_wb_resp) { + when(io.wb.resp.valid) { + state := Mux(!is_hit, s_outer_acquire, + Mux(pending_reads.orR, s_data_read, + Mux(!pending_writes.orR, s_inner_grant, + Mux(needs_more_put_data, s_wait_puts, s_data_write)))) + } + } is(s_inner_probe) { // Send probes when(io.inner.probe.ready) { From 002851f836b350b34e90aa2fb6eda44b5b18af7b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 18 Mar 2015 21:11:40 -0700 Subject: [PATCH 355/374] disentangle is_hit logic --- uncore/src/main/scala/cache.scala | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 146250bc..55fc544d 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -608,16 +608,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val pending_coh = Reg{ xact_meta.coh.clone } val present_puts = Reg(init=Bits(0, width = innerDataBeats)) present_puts := (present_puts | addPendingBitWhenHasData(io.inner.acquire)) - - val is_hit = (if(isLastLevelCache) - (xact.isBuiltInType(Acquire.putBlockType) || - xact_meta.coh.outer.isValid()) - else (xact_tag_match && xact_meta.coh.outer.isHit(xact.op_code()))) - val do_allocate = xact.allocate() - val needs_writeback = !xact_tag_match && do_allocate && - (xact_meta.coh.outer.requiresVoluntaryWriteback() || - xact_meta.coh.inner.requiresProbesOnVoluntaryWriteback()) val needs_more_put_data = !present_puts.andR + val do_allocate = xact.allocate() val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) @@ -861,7 +853,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { (xact.isBuiltInType(Acquire.putBlockType) || _tag_match && _coh.outer.isValid()) else (_tag_match && _coh.outer.isHit(xact.op_code()))) - val _needs_writeback = !_tag_match && do_allocate && (_coh.outer.requiresVoluntaryWriteback() || _coh.inner.requiresProbesOnVoluntaryWriteback()) @@ -881,11 +872,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } is(s_wb_req) { when(io.wb.req.ready) { state := s_wb_resp } } is(s_wb_resp) { - when(io.wb.resp.valid) { - state := Mux(!is_hit, s_outer_acquire, - Mux(pending_reads.orR, s_data_read, - Mux(!pending_writes.orR, s_inner_grant, - Mux(needs_more_put_data, s_wait_puts, s_data_write)))) + when(io.wb.resp.valid) { + val _skip_outer_acquire = Bool(isLastLevelCache) && xact.isBuiltInType(Acquire.putBlockType) + state := Mux(!_skip_outer_acquire, s_outer_acquire, + Mux(needs_more_put_data, s_wait_puts, s_data_write)) } } is(s_inner_probe) { @@ -907,7 +897,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } } when(release_count === UInt(0)) { - state := Mux(!is_hit, s_outer_acquire, + val _skip_outer_acquire = (if(isLastLevelCache) + (xact.isBuiltInType(Acquire.putBlockType) || + xact_meta.coh.outer.isValid()) + else xact_meta.coh.outer.isHit(xact.op_code())) + state := Mux(!_skip_outer_acquire, s_outer_acquire, Mux(pending_reads.orR, s_data_read, Mux(!pending_writes.orR, s_inner_grant, Mux(needs_more_put_data, s_wait_puts, s_data_write)))) @@ -968,7 +962,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { is(s_inner_grant) { when(ignt_q.io.count === UInt(0) || (ignt_data_done && ignt_q.io.count === UInt(1))) { - val meta_dirty = !is_hit || pending_coh_on_ignt != xact_meta.coh + val meta_dirty = !xact_tag_match || pending_coh_on_ignt != xact_meta.coh when(meta_dirty) { pending_coh := pending_coh_on_ignt } state := Mux(meta_dirty, s_meta_write, Mux(ifin_cnt > UInt(0) || io.ignt().requiresAck(), From 004ad11af6fa5636ac046871284347a156a72444 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 18 Mar 2015 22:14:41 -0700 Subject: [PATCH 356/374] cleanup pending signals --- uncore/src/main/scala/cache.scala | 23 ++++++++++------------- uncore/src/main/scala/uncore.scala | 14 +++++--------- 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 55fc544d..ed10e3c5 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -606,9 +606,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } val pending_coh = Reg{ xact_meta.coh.clone } - val present_puts = Reg(init=Bits(0, width = innerDataBeats)) - present_puts := (present_puts | addPendingBitWhenHasData(io.inner.acquire)) - val needs_more_put_data = !present_puts.andR + val pending_puts = Reg(init=Bits(0, width = innerDataBeats)) + pending_puts := (pending_puts & dropPendingBitWhenHasData(io.inner.acquire)) + val needs_more_put_data = pending_puts.orR val do_allocate = xact.allocate() val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) @@ -648,9 +648,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) pending_reads := (pending_reads | - addPendingBitWhenWmaskIsNotFull(io.inner.acquire)) & + addPendingBitWhenGetOrAtomic(io.inner.acquire)) & (dropPendingBit(io.data.read) & - dropPendingBitWhenWmaskIsFull(io.inner.acquire) & dropPendingBitWhenHasData(io.inner.release) & dropPendingBitWhenHasData(io.outer.grant)) val curr_read_beat = PriorityEncoder(pending_reads) @@ -824,14 +823,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { xact := io.iacq() xact.data := UInt(0) wmask_buffer.foreach { w => w := UInt(0) } - present_puts := Mux(io.iacq().isBuiltInType(Acquire.putBlockType), - addPendingBitWhenHasData(io.inner.acquire), - SInt(-1, width = innerDataBeats)).toUInt - pending_reads := Mux(io.iacq().isBuiltInType(Acquire.putBlockType), - UInt(0), - Mux(io.iacq().isSubBlockType(), - addPendingBitWhenWmaskIsNotFull(io.inner.acquire), - SInt(-1, width = innerDataBeats)).toUInt) + pending_puts := Mux(io.iacq().isBuiltInType(Acquire.putBlockType), + dropPendingBitWhenHasData(io.inner.acquire), + UInt(0)) + pending_reads := Mux(io.iacq().isBuiltInType(), + addPendingBitWhenGetOrAtomic(io.inner.acquire), + SInt(-1, width = innerDataBeats)).toUInt pending_writes := addPendingBitWhenHasData(io.inner.acquire) pending_resps := UInt(0) pending_ignt_data := UInt(0) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index cfd33567..de55c0e2 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -160,14 +160,10 @@ abstract class XactTracker extends CoherenceAgentModule { ~UIntToOH(in.bits.payload.addr_beat) } - //TODO | with existing wmask_buffer? - def addPendingBitWhenWmaskIsNotFull(in: DecoupledIO[LogicalNetworkIO[Acquire]]) = { - Fill(in.bits.payload.tlDataBeats, in.fire() && !in.bits.payload.wmask().andR) & - UIntToOH(in.bits.payload.addr_beat) - } - - def dropPendingBitWhenWmaskIsFull(in: DecoupledIO[LogicalNetworkIO[Acquire]]) = { - ~Fill(in.bits.payload.tlDataBeats, in.fire() && in.bits.payload.wmask().andR) | - ~UIntToOH(in.bits.payload.addr_beat) + def addPendingBitWhenGetOrAtomic(in: DecoupledIO[LogicalNetworkIO[Acquire]]) = { + val a = in.bits.payload + Fill(a.tlDataBeats, in.fire() && a.isBuiltInType() && + (a.is(Acquire.getType) || a.is(Acquire.getBlockType) || a.is(Acquire.putAtomicType))) & + UIntToOH(a.addr_beat) } } From 3cf033180f2980a34b07016a4197d949e57a9b5d Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 18 Mar 2015 22:41:09 -0700 Subject: [PATCH 357/374] pending read fix --- uncore/src/main/scala/cache.scala | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index ed10e3c5..5f656f31 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -826,9 +826,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_puts := Mux(io.iacq().isBuiltInType(Acquire.putBlockType), dropPendingBitWhenHasData(io.inner.acquire), UInt(0)) - pending_reads := Mux(io.iacq().isBuiltInType(), - addPendingBitWhenGetOrAtomic(io.inner.acquire), - SInt(-1, width = innerDataBeats)).toUInt + pending_reads := Mux(io.iacq().isBuiltInType(Acquire.getBlockType) || + !io.iacq().isBuiltInType(), + SInt(-1, width = innerDataBeats), + addPendingBitWhenGetOrAtomic(io.inner.acquire)).toUInt pending_writes := addPendingBitWhenHasData(io.inner.acquire) pending_resps := UInt(0) pending_ignt_data := UInt(0) From 4176edaa34956cc1ce08d108d790aebf267bdd1f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 26 Mar 2015 10:17:51 -0700 Subject: [PATCH 358/374] clean up tracker allocation --- uncore/src/main/scala/broadcast.scala | 38 +++++++++++++-------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 2564140b..64a96cf3 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -48,28 +48,24 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent when (sdq_enq) { sdq(sdq_alloc_id) := io.iacq().data } // Handle acquire transaction initiation - val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) + val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) + val acquireConflicts = Vec(trackerList.map(_.io.has_acquire_conflict)).toBits + val acquireMatches = Vec(trackerList.map(_.io.has_acquire_match)).toBits + val acquireReadys = Vec(trackerAcquireIOs.map(_.ready)).toBits + val acquire_idx = Mux(acquireMatches.orR, + PriorityEncoder(acquireMatches), + PriorityEncoder(acquireReadys)) - val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) - alloc_arb.io.out.ready := Bool(true) - trackerAcquireIOs.zip(alloc_arb.io.in).foreach { - case(tracker, arb) => arb.valid := tracker.ready - } - val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} - - val acquireMatchList = trackerList.map(_.io.has_acquire_match) - val any_acquire_matches = acquireMatchList.reduce(_||_) - val match_idx = Vec(acquireMatchList).indexWhere{b: Bool => b} - - val acquire_idx = Mux(any_acquire_matches, match_idx, alloc_idx) - val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && !block_acquires && sdq_rdy + val block_acquires = acquireConflicts.orR || !sdq_rdy + io.inner.acquire.ready := acquireReadys.orR && !block_acquires trackerAcquireIOs.zipWithIndex.foreach { case(tracker, i) => tracker.bits := io.inner.acquire.bits tracker.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits - tracker.valid := io.inner.acquire.valid && !block_acquires && (acquire_idx === UInt(i)) + tracker.valid := io.inner.acquire.valid && + !block_acquires && + (acquire_idx === UInt(i)) } // Queue to store impending Voluntary Release data @@ -80,17 +76,21 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent when(vwbdq_enq) { vwbdq(rel_data_cnt) := io.irel().data } // Handle releases, which might be voluntary and might have data - val release_idx = Vec(trackerList.map(_.io.has_release_match)).indexWhere{b: Bool => b} val trackerReleaseIOs = trackerList.map(_.io.inner.release) + val releaseReadys = Vec(trackerReleaseIOs.map(_.ready)).toBits + val releaseMatches = Vec(trackerList.map(_.io.has_release_match)).toBits + val release_idx = PriorityEncoder(releaseMatches) + io.inner.release.ready := releaseReadys(release_idx) trackerReleaseIOs.zipWithIndex.foreach { case(tracker, i) => + tracker.valid := io.inner.release.valid && (release_idx === UInt(i)) tracker.bits := io.inner.release.bits tracker.bits.payload.data := DataQueueLocation(rel_data_cnt, (if(i < nReleaseTransactors) inVolWBQueue else inClientReleaseQueue)).toBits - tracker.valid := io.inner.release.valid && (release_idx === UInt(i)) } - io.inner.release.ready := Vec(trackerReleaseIOs.map(_.ready)).read(release_idx) + assert(!(io.inner.release.valid && !releaseMatches.orR), + "Non-voluntary release should always have a Tracker waiting for it.") // Wire probe requests and grant reply to clients, finish acks from clients // Note that we bypass the Grant data subbundles From b7af610569ce10c952519359f63cbb10509c2b43 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 26 Mar 2015 11:29:04 -0700 Subject: [PATCH 359/374] broadcast hub bugfix --- uncore/src/main/scala/broadcast.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 64a96cf3..ce72f625 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -229,9 +229,9 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact val curr_p_id = PriorityEncoder(pending_probes) val full_sharers = coh.full() val probe_self = io.inner.acquire.bits.payload.requiresSelfProbe() - val mask_self = Mux(probe_self, - full_sharers | UInt(UInt(1) << xact_src, width = nCoherentClients), - full_sharers & ~UInt(UInt(1) << xact_src, width = nCoherentClients)) + val mask_self_true = UInt(UInt(1) << io.inner.acquire.bits.header.src, width = nCoherentClients) + val mask_self_false = ~UInt(UInt(1) << io.inner.acquire.bits.header.src, width = nCoherentClients) + val mask_self = Mux(probe_self, full_sharers | mask_self_true, full_sharers & mask_self_false) val mask_incoherent = mask_self & ~io.incoherent.toBits val collect_iacq_data = Reg(init=Bool(false)) From 8959b2e81a0ae6fb198a37a72ebc25d19af4b1ab Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 26 Mar 2015 13:29:52 -0700 Subject: [PATCH 360/374] TileLinkEnqueuer --- uncore/src/main/scala/tilelink.scala | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 42c17ca4..0aff9aab 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -505,6 +505,29 @@ object HeaderlessTileLinkIOWrapper { } } +class TileLinkEnqueuer(depths: (Int, Int, Int, Int, Int)) extends Module { + val io = new Bundle { + val client = new TileLinkIO().flip + val manager = new TileLinkIO + } + io.manager.acquire <> (if(depths._1 > 0) Queue(io.client.acquire, depths._1) else io.client.acquire) + io.client.probe <> (if(depths._2 > 0) Queue(io.manager.probe, depths._2) else io.manager.probe) + io.manager.release <> (if(depths._3 > 0) Queue(io.client.release, depths._3) else io.client.release) + io.client.grant <> (if(depths._4 > 0) Queue(io.manager.grant, depths._4) else io.manager.grant) + io.manager.finish <> (if(depths._5 > 0) Queue(io.client.finish, depths._5) else io.client.finish) +} + +object TileLinkEnqueuer { + def apply(in: TileLinkIO, depths: (Int, Int, Int, Int, Int))(p: Parameters): TileLinkIO = { + val t = Module(new TileLinkEnqueuer(depths))(p) + t.io.client <> in + t.io.manager + } + def apply(in: TileLinkIO, depth: Int)(p: Parameters): TileLinkIO = { + apply(in, (depth, depth, depth, depth, depth))(p) + } +} + abstract trait HasArbiterTypes { val arbN: Int type ManagerSourcedWithId = ManagerToClientChannel with HasClientTransactionId From c941f0a68ec202f96bade0bcf1a48ec8151785d5 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 27 Mar 2015 16:21:29 -0700 Subject: [PATCH 361/374] New virtual memory implementation (Sv39) --- uncore/src/main/scala/memserdes.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index e2d6df85..059fbdcf 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -7,8 +7,9 @@ import scala.math._ case object PAddrBits extends Field[Int] case object VAddrBits extends Field[Int] case object PgIdxBits extends Field[Int] +case object PgLevels extends Field[Int] +case object PgLevelBits extends Field[Int] case object ASIdBits extends Field[Int] -case object PermBits extends Field[Int] case object PPNBits extends Field[Int] case object VPNBits extends Field[Int] From ced627f00a94ac772fc3bfa8e19ddecfae203769 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 19 Mar 2015 01:11:34 -0700 Subject: [PATCH 362/374] slight mod to pending_puts cleaner state transition logic --- uncore/src/main/scala/cache.scala | 46 +++++++++++++++---------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 5f656f31..a705f618 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -608,7 +608,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val pending_coh = Reg{ xact_meta.coh.clone } val pending_puts = Reg(init=Bits(0, width = innerDataBeats)) pending_puts := (pending_puts & dropPendingBitWhenHasData(io.inner.acquire)) - val needs_more_put_data = pending_puts.orR val do_allocate = xact.allocate() val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) @@ -816,6 +815,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.wb.req.bits.way_en := xact_way_en io.wb.req.bits.id := UInt(trackerId) + when(io.data.resp.valid) { + mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) + } + + def doneNextCycleHot(rdy: Bool, pending: UInt) = !pending.orR || rdy && PopCount(pending) === UInt(1) + def doneNextCycleCounter(rdy: Bool, pending: UInt) = pending === UInt(0) || rdy && pending === UInt(1) switch (state) { is(s_idle) { when(io.inner.acquire.valid) { @@ -865,7 +870,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { Mux(!_is_hit, s_outer_acquire, Mux(pending_reads.orR, s_data_read, Mux(!pending_writes.orR, s_inner_grant, - Mux(needs_more_put_data, s_wait_puts, s_data_write)))))) + Mux(pending_puts.orR, s_wait_puts, s_data_write)))))) } } is(s_wb_req) { when(io.wb.req.ready) { state := s_wb_resp } } @@ -873,7 +878,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.wb.resp.valid) { val _skip_outer_acquire = Bool(isLastLevelCache) && xact.isBuiltInType(Acquire.putBlockType) state := Mux(!_skip_outer_acquire, s_outer_acquire, - Mux(needs_more_put_data, s_wait_puts, s_data_write)) + Mux(pending_puts.orR, s_wait_puts, s_data_write)) } } is(s_inner_probe) { @@ -902,7 +907,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { state := Mux(!_skip_outer_acquire, s_outer_acquire, Mux(pending_reads.orR, s_data_read, Mux(!pending_writes.orR, s_inner_grant, - Mux(needs_more_put_data, s_wait_puts, s_data_write)))) + Mux(pending_puts.orR, s_wait_puts, s_data_write)))) } } is(s_outer_acquire) { when(oacq_data_done) { state := s_outer_grant } } @@ -921,7 +926,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { }.otherwise { state := Mux(pending_reads.orR, s_data_read, Mux(!pending_writes.orR, s_inner_grant, - Mux(needs_more_put_data, s_wait_puts, s_data_write))) + Mux(pending_puts.orR, s_wait_puts, s_data_write))) } } } @@ -930,36 +935,32 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.outer.finish.ready) { state := Mux(pending_reads.orR, s_data_read, Mux(!pending_writes.orR, s_inner_grant, - Mux(needs_more_put_data, s_wait_puts, s_data_write))) + Mux(pending_puts.orR, s_wait_puts, s_data_write))) } } is(s_data_read) { - when(io.data.resp.valid) { - mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) - } - when(io.data.read.ready) { - when(PopCount(pending_reads) <= UInt(1)) { state := s_data_resp } + when(doneNextCycleHot(io.data.read.ready, pending_reads)) { + state := s_data_resp } } is(s_data_resp) { - when(io.data.resp.valid) { - mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) - } - when(PopCount(pending_resps) === UInt(0) || - (io.data.resp.valid && PopCount(pending_resps) === UInt(1))) { - state := Mux(!pending_writes.orR, s_inner_grant, - Mux(needs_more_put_data, s_wait_puts, s_data_write)) + when(doneNextCycleHot(io.data.resp.valid, pending_resps)) { + state := Mux(!pending_writes.orR, s_inner_grant, + Mux(pending_puts.orR, s_wait_puts, s_data_write)) + } + } + is(s_wait_puts) { + when(doneNextCycleHot(io.inner.acquire.fire(), pending_puts)) { + state := s_data_write } } - is(s_wait_puts) { when(!needs_more_put_data) { state := s_data_write } } is(s_data_write) { when(io.data.write.ready) { when(PopCount(pending_writes) <= UInt(1)) { state := s_inner_grant } } } is(s_inner_grant) { - when(ignt_q.io.count === UInt(0) || - (ignt_data_done && ignt_q.io.count === UInt(1))) { + when(doneNextCycleCounter(ignt_data_done, ignt_q.io.count)) { val meta_dirty = !xact_tag_match || pending_coh_on_ignt != xact_meta.coh when(meta_dirty) { pending_coh := pending_coh_on_ignt } state := Mux(meta_dirty, s_meta_write, @@ -973,8 +974,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } } is(s_inner_finish) { - when(ifin_cnt === UInt(0) || - (io.inner.finish.valid && ifin_cnt === UInt(1))) { + when(doneNextCycleCounter(io.inner.finish.valid, ifin_cnt)) { state := s_idle } } From 9708d25dfff7b4a0a6381be398f00dad608a9436 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 24 Mar 2015 02:06:53 -0700 Subject: [PATCH 363/374] Restructure L2 state machine and utilize HeaderlessTileLinkIO --- uncore/src/main/scala/broadcast.scala | 43 +- uncore/src/main/scala/cache.scala | 709 ++++++++++++-------------- uncore/src/main/scala/htif.scala | 44 +- uncore/src/main/scala/memserdes.scala | 28 +- uncore/src/main/scala/metadata.scala | 4 +- uncore/src/main/scala/network.scala | 5 +- uncore/src/main/scala/tilelink.scala | 393 +++++++++++--- uncore/src/main/scala/uncore.scala | 93 ++-- uncore/src/main/scala/util.scala | 26 +- 9 files changed, 780 insertions(+), 565 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index ce72f625..98c8f0af 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -52,20 +52,17 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent val acquireConflicts = Vec(trackerList.map(_.io.has_acquire_conflict)).toBits val acquireMatches = Vec(trackerList.map(_.io.has_acquire_match)).toBits val acquireReadys = Vec(trackerAcquireIOs.map(_.ready)).toBits - val acquire_idx = Mux(acquireMatches.orR, + val acquire_idx = Mux(acquireMatches.orR, PriorityEncoder(acquireMatches), PriorityEncoder(acquireReadys)) val block_acquires = acquireConflicts.orR || !sdq_rdy - io.inner.acquire.ready := acquireReadys.orR && !block_acquires + io.inner.acquire.ready := acquireReadys.orR && !block_acquires trackerAcquireIOs.zipWithIndex.foreach { case(tracker, i) => tracker.bits := io.inner.acquire.bits - tracker.bits.payload.data := - DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits - tracker.valid := io.inner.acquire.valid && - !block_acquires && - (acquire_idx === UInt(i)) + tracker.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits + tracker.valid := io.inner.acquire.valid && !block_acquires && (acquire_idx === UInt(i)) } // Queue to store impending Voluntary Release data @@ -94,24 +91,24 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent // Wire probe requests and grant reply to clients, finish acks from clients // Note that we bypass the Grant data subbundles - io.inner.grant.bits.payload.data := io.outer.grant.bits.payload.data - io.inner.grant.bits.payload.addr_beat := io.outer.grant.bits.payload.addr_beat + io.inner.grant.bits.payload.data := io.outer.grant.bits.data + io.inner.grant.bits.payload.addr_beat := io.outer.grant.bits.addr_beat doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish)) // Create an arbiter for the one memory port - val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), + val outer_arb = Module(new HeaderlessUncachedTileLinkIOArbiter(trackerList.size), { case TLId => params(OuterTLId) case TLDataBits => internalDataBits }) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } // Get the pending data out of the store data queue - val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.payload.data) + val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.data) val is_in_sdq = outer_data_ptr.loc === inStoreQueue val free_sdq = io.outer.acquire.fire() && - io.outer.acquire.bits.payload.hasData() && + io.outer.acquire.bits.hasData() && outer_data_ptr.loc === inStoreQueue - io.outer.acquire.bits.payload.data := MuxLookup(outer_data_ptr.loc, io.irel().data, Array( + io.outer.acquire.bits.data := MuxLookup(outer_data_ptr.loc, io.irel().data, Array( inStoreQueue -> sdq(outer_data_ptr.idx), inVolWBQueue -> vwbdq(outer_data_ptr.idx))) io.outer <> outer_arb.io.out @@ -147,7 +144,6 @@ class BroadcastVoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends Broa io.outer.acquire.valid := Bool(false) io.outer.grant.ready := Bool(false) - io.outer.finish.valid := Bool(false) io.inner.acquire.ready := Bool(false) io.inner.probe.valid := Bool(false) io.inner.release.ready := Bool(false) @@ -159,11 +155,12 @@ class BroadcastVoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends Broa io.inner.grant.bits.payload := coh.makeGrant(xact, UInt(trackerId)) //TODO: Use io.outer.release instead? - io.outer.acquire.bits.payload := Bundle(PutBlock( - client_xact_id = UInt(trackerId), - addr_block = xact.addr_block, - addr_beat = oacq_data_cnt, - data = data_buffer(oacq_data_cnt)))(outerTLParams) + io.outer.acquire.bits := Bundle( + PutBlock( + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block, + addr_beat = oacq_data_cnt, + data = data_buffer(oacq_data_cnt)))(outerTLParams) when(collect_irel_data) { io.inner.release.ready := Bool(true) @@ -271,7 +268,7 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact addr_block = xact.addr_block))(outerTLParams) io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits.payload := outer_read //default + io.outer.acquire.bits := outer_read //default io.outer.grant.ready := Bool(false) io.inner.probe.valid := Bool(false) @@ -346,7 +343,7 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact when(io.inner.release.valid) { when(io.irel().hasData()) { io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_write_rel + io.outer.acquire.bits := outer_write_rel when(io.outer.acquire.ready) { when(oacq_data_done) { pending_ognt_ack := Bool(true) @@ -368,7 +365,7 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact } is(s_mem_write) { // Write data to outer memory io.outer.acquire.valid := !pending_ognt_ack || !collect_iacq_data || iacq_data_valid(oacq_data_cnt) - io.outer.acquire.bits.payload := outer_write_acq + io.outer.acquire.bits := outer_write_acq when(oacq_data_done) { pending_ognt_ack := Bool(true) state := Mux(pending_outer_read, s_mem_read, s_mem_resp) @@ -376,7 +373,7 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact } is(s_mem_read) { // Read data from outer memory (possibly what was just written) io.outer.acquire.valid := !pending_ognt_ack - io.outer.acquire.bits.payload := outer_read + io.outer.acquire.bits := outer_read when(io.outer.acquire.fire()) { state := s_mem_resp } } is(s_mem_resp) { // Wait to forward grants from outer memory diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index a705f618..05588203 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -2,6 +2,7 @@ package uncore import Chisel._ +import scala.reflect.ClassTag case object CacheName extends Field[String] case object NSets extends Field[Int] @@ -329,22 +330,13 @@ class L2HellaCacheBank(bankId: Int) extends HierarchicalCoherenceAgent require(isPow2(nSets)) require(isPow2(nWays)) + val meta = Module(new L2MetadataArray) // TODO: add delay knob + val data = Module(new L2DataArray(1)) val tshrfile = Module(new TSHRFile(bankId)) - - //TODO: Expose queue depths and data array pipeline cycles as parameters? - tshrfile.io.inner.acquire <> io.inner.acquire - tshrfile.io.inner.probe <> io.inner.probe - tshrfile.io.inner.release <> Queue(io.inner.release) - tshrfile.io.inner.grant <> io.inner.grant - tshrfile.io.inner.finish <> io.inner.finish - + tshrfile.io.inner <> io.inner io.outer <> tshrfile.io.outer io.incoherent <> tshrfile.io.incoherent - - val meta = Module(new L2MetadataArray) tshrfile.io.meta <> meta.io - - val data = Module(new L2DataArray(1)) tshrfile.io.data <> data.io } @@ -364,6 +356,7 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule Module(new L2AcquireTracker(id, bankId)) } + // WritebackUnit evicts data from L2, including invalidating L1s val wb = Module(new L2WritebackUnit(nTransactors, bankId)) doOutputArbitration(wb.io.wb.req, trackerList.map(_.io.wb.req)) doInputRouting(wb.io.wb.resp, trackerList.map(_.io.wb.resp)) @@ -373,21 +366,15 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule // Handle acquire transaction initiation val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) + val acquireConflicts = Vec(trackerList.map(_.io.has_acquire_conflict)).toBits + val acquireMatches = Vec(trackerList.map(_.io.has_acquire_match)).toBits + val acquireReadys = Vec(trackerAcquireIOs.map(_.ready)).toBits + val acquire_idx = Mux(acquireMatches.orR, + PriorityEncoder(acquireMatches), + PriorityEncoder(acquireReadys)) - val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) - alloc_arb.io.out.ready := Bool(true) - trackerAcquireIOs.zip(alloc_arb.io.in).foreach { - case(tracker, arb) => arb.valid := tracker.ready - } - val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} - - val acquireMatchList = trackerList.map(_.io.has_acquire_match) - val any_acquire_matches = acquireMatchList.reduce(_||_) - val match_idx = Vec(acquireMatchList).indexWhere{b: Bool => b} - - val acquire_idx = Mux(any_acquire_matches, match_idx, alloc_idx) - val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && !block_acquires + val block_acquires = acquireConflicts.orR + io.inner.acquire.ready := acquireReadys.orR && !block_acquires trackerAcquireIOs.zipWithIndex.foreach { case(tracker, i) => tracker.bits := io.inner.acquire.bits @@ -395,15 +382,18 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule } // Wire releases from clients - val release_idx = Vec(trackerList.map(_.io.has_release_match) :+ - wb.io.has_release_match).indexWhere{b: Bool => b} val trackerReleaseIOs = trackerList.map(_.io.inner.release) :+ wb.io.inner.release - trackerReleaseIOs.zipWithIndex.foreach { + val releaseReadys = Vec(trackerReleaseIOs.map(_.ready)).toBits + val releaseMatches = Vec(trackerList.map(_.io.has_release_match) :+ wb.io.has_release_match).toBits + val release_idx = PriorityEncoder(releaseMatches) + io.inner.release.ready := releaseReadys(release_idx) + trackerReleaseIOs.zipWithIndex.foreach { case(tracker, i) => tracker.bits := io.inner.release.bits tracker.valid := io.inner.release.valid && (release_idx === UInt(i)) } - io.inner.release.ready := Vec(trackerReleaseIOs.map(_.ready)).read(release_idx) + assert(!(io.inner.release.valid && !releaseMatches.orR), + "Non-voluntary release should always have a Tracker waiting for it.") // Wire probe requests and grant reply to clients, finish acks from clients doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe) :+ wb.io.inner.probe) @@ -412,7 +402,7 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule // Create an arbiter for the one memory port val outerList = trackerList.map(_.io.outer) :+ wb.io.outer - val outer_arb = Module(new TileLinkIOArbiterThatPassesId(outerList.size))(outerTLParams) + val outer_arb = Module(new HeaderlessTileLinkIOArbiter(outerList.size))(outerTLParams) outerList zip outer_arb.io.in map { case(out, arb) => out <> arb } io.outer <> outer_arb.io.out @@ -483,10 +473,10 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack val xact_src = Reg(io.inner.release.bits.header.src.clone) val xact = Reg(Bundle(new Release, { case TLId => params(InnerTLId); case TLDataBits => 0 })) val xact_tag_match = Reg{ Bool() } - val xact_meta = Reg{ new L2Metadata } + val xact_old_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) } - val coh = xact_meta.coh + val coh = xact_old_meta.coh val collect_irel_data = Reg(init=Bool(false)) val irel_data_valid = Reg(init=Bits(0, width = innerDataBeats)) @@ -501,7 +491,6 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack io.outer.probe.ready := Bool(false) io.outer.release.valid := Bool(false) io.outer.grant.ready := Bool(false) - io.outer.finish.valid := Bool(false) io.inner.acquire.ready := Bool(false) io.inner.probe.valid := Bool(false) io.inner.release.ready := Bool(false) @@ -529,8 +518,8 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en io.meta.write.bits.data.tag := xact.addr_block >> UInt(idxBits) - io.meta.write.bits.data.coh.inner := xact_meta.coh.inner.onRelease(xact, xact_src) - io.meta.write.bits.data.coh.outer := xact_meta.coh.outer.onHit(M_XWR) // WB is a write + io.meta.write.bits.data.coh.inner := xact_old_meta.coh.inner.onRelease(xact, xact_src) + io.meta.write.bits.data.coh.outer := xact_old_meta.coh.outer.onHit(M_XWR) // WB is a write io.wb.req.valid := Bool(false) when(collect_irel_data) { @@ -561,7 +550,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack is(s_meta_resp) { when(io.meta.resp.valid) { xact_tag_match := io.meta.resp.bits.tag_match - xact_meta := io.meta.resp.bits.meta + xact_old_meta := io.meta.resp.bits.meta xact_way_en := io.meta.resp.bits.way_en state := Mux(io.meta.resp.bits.tag_match, Mux(xact.hasData(), s_data_write, s_meta_write), @@ -595,101 +584,73 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val io = new L2XactTrackerIO - val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_inner_probe :: s_outer_acquire :: s_outer_grant :: s_outer_finish :: s_data_read :: s_data_resp :: s_wait_puts :: s_data_write :: s_inner_grant :: s_meta_write :: s_inner_finish :: Nil = Enum(UInt(), 16) + val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_inner_probe :: s_outer_acquire :: s_busy :: s_meta_write :: Nil = Enum(UInt(), 9) val state = Reg(init=s_idle) + // State holding transaction metadata val xact_src = Reg(io.inner.acquire.bits.header.src.clone) val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId) })) - val data_buffer = Vec.fill(innerDataBeats){ Reg(UInt(width = innerDataBits)) } - val wmask_buffer = Vec.fill(innerDataBeats){ Reg(Bits(width = innerDataBits/8)) } + val data_buffer = Vec.fill(innerDataBeats){ Reg(init=UInt(0, width = innerDataBits)) } + val wmask_buffer = Vec.fill(innerDataBeats){ Reg(init=UInt(0,width = innerDataBits/8)) } val xact_tag_match = Reg{ Bool() } - val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } - val pending_coh = Reg{ xact_meta.coh.clone } - val pending_puts = Reg(init=Bits(0, width = innerDataBeats)) - pending_puts := (pending_puts & dropPendingBitWhenHasData(io.inner.acquire)) - val do_allocate = xact.allocate() + val xact_old_meta = Reg{ new L2Metadata } + val pending_coh = Reg{ xact_old_meta.coh.clone } - val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) - val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) - val curr_probe_dst = PriorityEncoder(pending_probes) - val full_sharers = io.meta.resp.bits.meta.coh.inner.full() - val probe_self = xact.requiresSelfProbe() - val mask_self = Mux(probe_self, - full_sharers | UInt(UInt(1) << xact_src, width = nCoherentClients), - full_sharers & ~UInt(UInt(1) << xact_src, width = nCoherentClients)) - val mask_incoherent = mask_self & ~io.incoherent.toBits - - val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire, xact.addr_beat) - val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) - val pending_ofin = Reg{ io.outer.finish.bits.clone } - - val pending_ignt_data = Reg(init=Bits(0, width = innerDataBeats)) - pending_ignt_data := pending_ignt_data | - addPendingBitInternal(io.data.resp) | - addPendingBitWhenHasData(io.inner.release) | - addPendingBitWhenHasData(io.outer.grant) | - addPendingBitWhenHasData(io.inner.acquire) + // Secondary miss queue val ignt_q = Module(new Queue(new L2SecondaryMissInfo, nSecondaryMisses))(innerTLParams) - val (ignt_data_idx, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) - ignt_q.io.enq.valid := Bool(false) ignt_q.io.enq.bits.client_xact_id := io.iacq().client_xact_id ignt_q.io.enq.bits.addr_beat := io.iacq().addr_beat - ignt_q.io.deq.ready := ignt_data_done - - val ifin_cnt = Reg(init = UInt(0, width = log2Up(nSecondaryMisses+1))) - when(ignt_data_done) { - ifin_cnt := Mux(io.inner.finish.fire(), - Mux(io.ignt().requiresAck(), ifin_cnt, ifin_cnt - UInt(1)), - Mux(io.ignt().requiresAck(), ifin_cnt + UInt(1), ifin_cnt)) - } .elsewhen(io.inner.finish.fire()) { ifin_cnt := ifin_cnt - UInt(1) } + // TODO add ignt.dst <- iacq.src + // State holding progress made on processing this transaction + val iacq_data_done = + connectIncomingDataBeatCounter(io.inner.acquire) + val pending_irels = + connectTwoWayBeatCounter(nCoherentClients, io.inner.probe, io.inner.release)._1 + val (pending_ognts, oacq_data_idx, oacq_data_done, ognt_data_idx, ognt_data_done) = + connectHeaderlessTwoWayBeatCounter(1, io.outer.acquire, io.outer.grant, xact.addr_beat) + val (ignt_data_idx, ignt_data_done) = + connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) + val pending_ifins = + connectTwoWayBeatCounter(nSecondaryMisses, io.inner.grant, io.inner.finish, (g: Grant) => g.requiresAck())._1 + val pending_puts = Reg(init=Bits(0, width = innerDataBeats)) + val pending_iprbs = Reg(init = Bits(0, width = nCoherentClients)) val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) - pending_reads := (pending_reads | - addPendingBitWhenGetOrAtomic(io.inner.acquire)) & - (dropPendingBit(io.data.read) & - dropPendingBitWhenHasData(io.inner.release) & - dropPendingBitWhenHasData(io.outer.grant)) - val curr_read_beat = PriorityEncoder(pending_reads) - val pending_writes = Reg(init=Bits(0, width = innerDataBeats)) - pending_writes := (pending_writes | - addPendingBitWhenHasData(io.inner.acquire) | - addPendingBitWhenHasData(io.inner.release) | - addPendingBitWhenHasData(io.outer.grant)) & - dropPendingBit(io.data.write) - val curr_write_beat = PriorityEncoder(pending_writes) - val pending_resps = Reg(init=Bits(0, width = innerDataBeats)) - pending_resps := (pending_resps | - addPendingBitInternal(io.data.read)) & - dropPendingBitInternal(io.data.resp) + val pending_ignt_data = Reg(init=Bits(0, width = innerDataBeats)) + val pending_meta_write = Reg{ Bool() } - val pending_coh_on_hit = HierarchicalMetadata( - io.meta.resp.bits.meta.coh.inner, - io.meta.resp.bits.meta.coh.outer.onHit(xact.op_code())) - val pending_icoh_on_irel = pending_coh.inner.onRelease( - incoming = io.irel(), - src = io.inner.release.bits.header.src) - val pending_ocoh_on_irel = pending_coh.outer.onHit(M_XWR) // WB is a write - val pending_coh_on_ognt = HierarchicalMetadata( - ManagerMetadata.onReset, - pending_coh.outer.onGrant(io.ognt(), xact.op_code())) - val pending_coh_on_ignt = HierarchicalMetadata( - pending_coh.inner.onGrant( - outgoing = io.ignt(), - dst = io.inner.grant.bits.header.dst), - pending_coh.outer) - val pending_ofin_on_ognt = io.ognt().makeFinish() + val all_pending_done = + !(pending_reads.orR || + pending_writes.orR || + pending_resps.orR || + pending_puts.orR || + pending_ognts || + ignt_q.io.count > UInt(0) || + //pending_meta_write || // Has own state: s_meta_write + pending_ifins) - val amo_result = xact.data + // Provide a single ALU per tracker to merge Puts and AMOs with data being + // refilled, written back, or extant in the cache val amoalu = Module(new AMOALU) amoalu.io.addr := xact.addr() amoalu.io.cmd := xact.op_code() amoalu.io.typ := xact.op_size() - amoalu.io.lhs := io.data.resp.bits.data //default - amoalu.io.rhs := data_buffer.head // default + amoalu.io.lhs := io.data.resp.bits.data // default, overwritten by calls to mergeData + amoalu.io.rhs := data_buffer.head // default, overwritten by calls to mergeData + val amo_result = xact.data // Reuse xact buffer space to store AMO result + + // Utility functions for updating the data and metadata that will be kept in + // the cache or granted to the original requestor after this transaction: + + def updatePendingCohWhen(flag: Bool, next: HierarchicalMetadata) { + when(flag && pending_coh != next) { + pending_meta_write := Bool(true) + pending_coh := next + } + } def mergeData(dataBits: Int)(beat: UInt, incoming: UInt) { val old_data = incoming // Refilled, written back, or de-cached data @@ -704,104 +665,182 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { wmask_buffer(beat) := SInt(-1) when(xact.is(Acquire.putAtomicType) && xact.addr_beat === beat) { amo_result := old_data } } - val mergeDataInternal = mergeData(rowBits) _ - val mergeDataInner = mergeData(innerDataBits) _ - val mergeDataOuter = mergeData(outerDataBits) _ + def mergeDataInternal[T <: HasL2Data with HasL2BeatAddr](in: ValidIO[T]) { + when(in.valid) { mergeData(rowBits)(in.bits.addr_beat, in.bits.data) } + } + def mergeDataInner[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[LogicalNetworkIO[T]]) { + when(in.fire() && in.bits.payload.hasData()) { + mergeData(innerDataBits)(in.bits.payload.addr_beat, in.bits.payload.data) + } + } + def mergeDataOuter[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[T]) { + when(in.fire() && in.bits.hasData()) { + mergeData(outerDataBits)(in.bits.addr_beat, in.bits.data) + } + } + // Actual transaction processing logic begins here: + // + // First, take care of accpeting new requires or secondary misses + // For now, the only allowed secondary miss types are Gets-under-Get + // and Puts-under-Put from the same client val can_merge_iacq_get = (xact.isBuiltInType(Acquire.getType) && io.iacq().isBuiltInType(Acquire.getType)) && - (xact_src === io.inner.acquire.bits.header.src) && - xact.conflicts(io.iacq()) && - Vec(s_meta_read, s_meta_resp, s_wb_req, s_wb_resp, - s_inner_probe, s_outer_acquire, s_outer_grant, - s_outer_finish).contains(state) && - do_allocate && - ignt_q.io.enq.ready + xact_src === io.inner.acquire.bits.header.src && //TODO + xact.conflicts(io.iacq()) && + state != s_idle && state != s_meta_write && + !all_pending_done && + xact.allocate() && + !io.inner.release.fire() && + !io.outer.grant.fire() && + !io.data.resp.valid && + ignt_q.io.enq.ready + // This logic also allows the tail beats of a PutBlock to be merged in val can_merge_iacq_put = ((xact.isBuiltInType(Acquire.putType) && io.iacq().isBuiltInType(Acquire.putType)) || (xact.isBuiltInType(Acquire.putBlockType) && io.iacq().isBuiltInType(Acquire.putBlockType))) && - (xact_src === io.inner.acquire.bits.header.src) && - (xact.client_xact_id === io.iacq().client_xact_id) && - xact.conflicts(io.iacq()) && - Vec(s_meta_read, s_meta_resp, s_wb_req, s_wb_resp, - s_inner_probe, s_outer_acquire, s_outer_grant, - s_outer_finish, s_data_read, - s_data_resp).contains(state) && - do_allocate && - ignt_q.io.enq.ready - - val in_same_set = xact.addr_block(idxMSB,idxLSB) === - io.iacq().addr_block(idxMSB,idxLSB) - io.has_release_match := xact.conflicts(io.irel()) && - !io.irel().isVoluntary() && - (state === s_inner_probe) - io.has_acquire_match := can_merge_iacq_put || can_merge_iacq_get - io.has_acquire_conflict := (xact.conflicts(io.iacq()) || in_same_set) && - (state != s_idle) && - !io.has_acquire_match - - // If we're allocating in this cache, we can use the current metadata - // to make an appropriate custom Acquire, otherwise we copy over the - // built-in Acquire from the inner TL to the outer TL - io.outer.acquire.valid := state === s_outer_acquire - io.outer.acquire.bits.payload := Mux(do_allocate, - xact_meta.coh.outer.makeAcquire( - client_xact_id = UInt(trackerId), - addr_block = xact.addr_block, - op_code = xact.op_code()), - Bundle(Acquire(xact))(outerTLParams)) - io.outer.acquire.bits.header.src := UInt(bankId) - io.outer.probe.ready := Bool(false) - io.outer.release.valid := Bool(false) - io.outer.grant.ready := state === s_outer_grant - io.outer.finish.valid := state === s_outer_finish - io.outer.finish.bits := pending_ofin - - io.inner.probe.valid := state === s_inner_probe && pending_probes.orR - io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_probe_dst - io.inner.probe.bits.payload := pending_coh.inner.makeProbe(xact) - - io.inner.grant.valid := state === s_inner_grant && - ignt_q.io.deq.valid && - (!io.ignt().hasData() || - pending_ignt_data(ignt_data_idx)) - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := pending_coh.inner.makeGrant( - acq = xact, - manager_xact_id = UInt(trackerId), - addr_beat = ignt_data_idx, - data = Mux(xact.is(Acquire.putAtomicType), - amo_result, - data_buffer(ignt_data_idx))) - io.ignt().client_xact_id := ignt_q.io.deq.bits.client_xact_id + xact_src === io.inner.acquire.bits.header.src && //TODO + xact.conflicts(io.iacq()) && + state != s_idle && state != s_meta_write && + !all_pending_done && + xact.allocate() && + !io.inner.release.fire() && + !io.outer.grant.fire() && + !io.data.resp.valid && + ignt_q.io.enq.ready io.inner.acquire.ready := state === s_idle || can_merge_iacq_put || can_merge_iacq_get - io.inner.release.ready := state === s_inner_probe - io.inner.finish.ready := Vec(s_inner_finish, s_meta_write, s_inner_grant, - s_data_write, s_wait_puts, s_data_resp).contains(state) - io.data.read.valid := state === s_data_read && pending_reads.orR + // Track whether any beats are missing from a PutBlock + pending_puts := (pending_puts & dropPendingBitWhenBeatHasData(io.inner.acquire)) + + // Track which clients yet need to be probed and make Probe message + pending_iprbs := pending_iprbs & dropPendingBitAtDest(io.inner.probe) + val curr_probe_dst = PriorityEncoder(pending_iprbs) + io.inner.probe.valid := state === s_inner_probe && pending_iprbs.orR + io.inner.probe.bits.header.src := UInt(bankId) + io.inner.probe.bits.header.dst := curr_probe_dst + io.inner.probe.bits.payload := pending_coh.inner.makeProbe(xact) + + // Handle incoming releases from clients, which may reduce sharer counts + // and/or write back dirty data + io.inner.release.ready := state === s_inner_probe + val pending_coh_on_irel = HierarchicalMetadata( + pending_coh.inner.onRelease( // Drop sharer + incoming = io.irel(), + src = io.inner.release.bits.header.src), + Mux(io.irel().hasData(), // Dirty writeback + pending_coh.outer.onHit(M_XWR), + pending_coh.outer)) + updatePendingCohWhen(io.inner.release.fire(), pending_coh_on_irel) + mergeDataInner(io.inner.release) + + // The following subtrascation handles misses or coherence permission + // upgrades by initiating a new transaction in the outer memory: + // + // If we're allocating in this cache, we can use the current metadata + // to make an appropriate custom Acquire, otherwise we copy over the + // built-in Acquire from the inner TL to the outer TL + io.outer.acquire.valid := state === s_outer_acquire + io.outer.acquire.bits := Mux( + xact.allocate(), + xact_old_meta.coh.outer.makeAcquire( + client_xact_id = UInt(0), + addr_block = xact.addr_block, + op_code = xact.op_code()), + Bundle(Acquire(xact))(outerTLParams)) + + // Probes from the outer memory system are handled in the WritebackUnit + io.outer.probe.ready := Bool(false) + io.outer.release.valid := Bool(false) + + io.outer.grant.ready := state === s_busy + val pending_coh_on_ognt = HierarchicalMetadata( + ManagerMetadata.onReset, + pending_coh.outer.onGrant(io.outer.grant.bits, xact.op_code())) + updatePendingCohWhen(ognt_data_done, pending_coh_on_ognt) + mergeDataOuter(io.outer.grant) + + // Going back to the original inner transaction, we can issue a Grant as + // soon as the data is released, granted, put, or read from the cache + pending_ignt_data := pending_ignt_data | + addPendingBitWhenBeatHasData(io.inner.release) | + addPendingBitWhenBeatHasData(io.outer.grant) | + addPendingBitInternal(io.data.resp) + ignt_q.io.enq.valid := iacq_data_done + ignt_q.io.deq.ready := ignt_data_done + // Make the Grant message using the data stored in the secondary miss queue + io.inner.grant.valid := state === s_busy && + ignt_q.io.deq.valid && + (!io.ignt().hasData() || + pending_ignt_data(ignt_data_idx)) + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := xact_src // TODO: ignt_q.io.deq.bits.src + io.inner.grant.bits.payload := pending_coh.inner.makeGrant( + acq = xact, + manager_xact_id = UInt(trackerId), + addr_beat = ignt_data_idx, + data = Mux(xact.is(Acquire.putAtomicType), + amo_result, + data_buffer(ignt_data_idx))) + // TODO: improve the ManagerMetadata.makeGrant to deal with possibility of + // multiple client transaction ids from merged secondary misses + io.ignt().client_xact_id := ignt_q.io.deq.bits.client_xact_id + + val pending_coh_on_ignt = HierarchicalMetadata( + pending_coh.inner.onGrant( + outgoing = io.ignt(), + dst = io.inner.grant.bits.header.dst), + pending_coh.outer) + updatePendingCohWhen(io.inner.grant.fire(), pending_coh_on_ignt) + + // We must wait for as many Finishes as we sent Grants + io.inner.finish.ready := state === s_busy + + pending_reads := (pending_reads & + dropPendingBit(io.data.read) & + dropPendingBitWhenBeatHasData(io.inner.release) & + dropPendingBitWhenBeatHasData(io.outer.grant)) | + addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire) + val curr_read_beat = PriorityEncoder(pending_reads) + io.data.read.valid := state === s_busy && + pending_reads.orR && + !pending_ognts io.data.read.bits.id := UInt(trackerId) io.data.read.bits.way_en := xact_way_en io.data.read.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) io.data.read.bits.addr_beat := curr_read_beat - io.data.write.valid := state === s_data_write && pending_writes.orR + + pending_resps := (pending_resps & dropPendingBitInternal(io.data.resp)) | + addPendingBitInternal(io.data.read) + mergeDataInternal(io.data.resp) + + pending_writes := (pending_writes & dropPendingBit(io.data.write)) | + addPendingBitWhenBeatHasData(io.inner.acquire) | + addPendingBitWhenBeatHasData(io.inner.release) | + addPendingBitWhenBeatHasData(io.outer.grant) + val curr_write_beat = PriorityEncoder(pending_writes) + io.data.write.valid := state === s_busy && + pending_writes.orR && + !pending_ognts && + !pending_reads(curr_write_beat) && + !pending_resps(curr_write_beat) io.data.write.bits.id := UInt(trackerId) io.data.write.bits.way_en := xact_way_en io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) io.data.write.bits.addr_beat := curr_write_beat io.data.write.bits.wmask := wmask_buffer(curr_write_beat) io.data.write.bits.data := data_buffer(curr_write_beat) + io.meta.read.valid := state === s_meta_read io.meta.read.bits.id := UInt(trackerId) io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) io.meta.read.bits.tag := xact.addr_block >> UInt(idxBits) + io.meta.write.valid := state === s_meta_write io.meta.write.bits.id := UInt(trackerId) io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) @@ -810,188 +849,106 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.meta.write.bits.data.coh := pending_coh io.wb.req.valid := state === s_wb_req - io.wb.req.bits.addr_block := Cat(xact_meta.tag, xact.addr_block(idxMSB,idxLSB)) - io.wb.req.bits.coh := xact_meta.coh + io.wb.req.bits.addr_block := Cat(xact_old_meta.tag, xact.addr_block(idxMSB,idxLSB)) + io.wb.req.bits.coh := xact_old_meta.coh io.wb.req.bits.way_en := xact_way_en io.wb.req.bits.id := UInt(trackerId) - when(io.data.resp.valid) { - mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) + // Handling of secondary misses (Gets and Puts only for now) + when(io.inner.acquire.fire() && io.iacq().hasData()) { // state <= s_meta_wrtie + val beat = io.iacq().addr_beat + val wmask = io.iacq().wmask() + val full = FillInterleaved(8, wmask) + data_buffer(beat) := (~full & data_buffer(beat)) | (full & io.iacq().data) + wmask_buffer(beat) := wmask | Mux(state === s_idle, Bits(0), wmask_buffer(beat)) } - def doneNextCycleHot(rdy: Bool, pending: UInt) = !pending.orR || rdy && PopCount(pending) === UInt(1) - def doneNextCycleCounter(rdy: Bool, pending: UInt) = pending === UInt(0) || rdy && pending === UInt(1) - switch (state) { - is(s_idle) { - when(io.inner.acquire.valid) { - xact_src := io.inner.acquire.bits.header.src - xact := io.iacq() - xact.data := UInt(0) - wmask_buffer.foreach { w => w := UInt(0) } - pending_puts := Mux(io.iacq().isBuiltInType(Acquire.putBlockType), - dropPendingBitWhenHasData(io.inner.acquire), - UInt(0)) - pending_reads := Mux(io.iacq().isBuiltInType(Acquire.getBlockType) || - !io.iacq().isBuiltInType(), - SInt(-1, width = innerDataBeats), - addPendingBitWhenGetOrAtomic(io.inner.acquire)).toUInt - pending_writes := addPendingBitWhenHasData(io.inner.acquire) - pending_resps := UInt(0) - pending_ignt_data := UInt(0) - ifin_cnt := UInt(0) - ignt_q.io.enq.valid := Bool(true) - state := s_meta_read - } - } - is(s_meta_read) { when(io.meta.read.ready) { state := s_meta_resp } } - is(s_meta_resp) { - when(io.meta.resp.valid) { - xact_tag_match := io.meta.resp.bits.tag_match - xact_meta := io.meta.resp.bits.meta - xact_way_en := io.meta.resp.bits.way_en - pending_coh := io.meta.resp.bits.meta.coh - val _coh = io.meta.resp.bits.meta.coh - val _tag_match = io.meta.resp.bits.tag_match - val _is_hit = (if(isLastLevelCache) - (xact.isBuiltInType(Acquire.putBlockType) || - _tag_match && _coh.outer.isValid()) - else (_tag_match && _coh.outer.isHit(xact.op_code()))) - val _needs_writeback = !_tag_match && do_allocate && - (_coh.outer.requiresVoluntaryWriteback() || - _coh.inner.requiresProbesOnVoluntaryWriteback()) - val _needs_inner_probes = _tag_match && _coh.inner.requiresProbes(xact) - when(_is_hit) { pending_coh := pending_coh_on_hit } - when(_needs_inner_probes) { - pending_probes := mask_incoherent - release_count := PopCount(mask_incoherent) - } - state := Mux(_needs_writeback, s_wb_req, - Mux(_needs_inner_probes, s_inner_probe, - Mux(!_is_hit, s_outer_acquire, - Mux(pending_reads.orR, s_data_read, - Mux(!pending_writes.orR, s_inner_grant, - Mux(pending_puts.orR, s_wait_puts, s_data_write)))))) - } - } - is(s_wb_req) { when(io.wb.req.ready) { state := s_wb_resp } } - is(s_wb_resp) { - when(io.wb.resp.valid) { - val _skip_outer_acquire = Bool(isLastLevelCache) && xact.isBuiltInType(Acquire.putBlockType) - state := Mux(!_skip_outer_acquire, s_outer_acquire, - Mux(pending_puts.orR, s_wait_puts, s_data_write)) - } - } - is(s_inner_probe) { - // Send probes - when(io.inner.probe.ready) { - pending_probes := pending_probes & ~UIntToOH(curr_probe_dst) - } - // Handle releases, which may have data being written back - when(io.inner.release.valid) { - pending_coh.inner := pending_icoh_on_irel - // Handle released dirty data - when(io.irel().hasData()) { - pending_coh.outer := pending_ocoh_on_irel - mergeDataInner(io.irel().addr_beat, io.irel().data) - } - // We don't decrement release_count until we've received all the data beats. - when(!io.irel().hasMultibeatData() || irel_data_done) { - release_count := release_count - UInt(1) - } - } - when(release_count === UInt(0)) { - val _skip_outer_acquire = (if(isLastLevelCache) - (xact.isBuiltInType(Acquire.putBlockType) || - xact_meta.coh.outer.isValid()) - else xact_meta.coh.outer.isHit(xact.op_code())) - state := Mux(!_skip_outer_acquire, s_outer_acquire, - Mux(pending_reads.orR, s_data_read, - Mux(!pending_writes.orR, s_inner_grant, - Mux(pending_puts.orR, s_wait_puts, s_data_write)))) - } - } - is(s_outer_acquire) { when(oacq_data_done) { state := s_outer_grant } } - is(s_outer_grant) { - when(io.outer.grant.valid) { - when(io.ognt().hasData()) { - mergeDataOuter(io.ognt().addr_beat, io.ognt().data) - } - when(ognt_data_done) { - pending_coh := pending_coh_on_ognt - when(io.ognt().requiresAck()) { - pending_ofin.payload := pending_ofin_on_ognt - pending_ofin.header.dst := io.outer.grant.bits.header.src - pending_ofin.header.src := UInt(bankId) - state := s_outer_finish - }.otherwise { - state := Mux(pending_reads.orR, s_data_read, - Mux(!pending_writes.orR, s_inner_grant, - Mux(pending_puts.orR, s_wait_puts, s_data_write))) - } - } - } - } - is(s_outer_finish) { - when(io.outer.finish.ready) { - state := Mux(pending_reads.orR, s_data_read, - Mux(!pending_writes.orR, s_inner_grant, - Mux(pending_puts.orR, s_wait_puts, s_data_write))) - } - } - is(s_data_read) { - when(doneNextCycleHot(io.data.read.ready, pending_reads)) { - state := s_data_resp - } - } - is(s_data_resp) { - when(doneNextCycleHot(io.data.resp.valid, pending_resps)) { - state := Mux(!pending_writes.orR, s_inner_grant, - Mux(pending_puts.orR, s_wait_puts, s_data_write)) - } - } - is(s_wait_puts) { - when(doneNextCycleHot(io.inner.acquire.fire(), pending_puts)) { - state := s_data_write - } - } - is(s_data_write) { - when(io.data.write.ready) { - when(PopCount(pending_writes) <= UInt(1)) { state := s_inner_grant } - } - } - is(s_inner_grant) { - when(doneNextCycleCounter(ignt_data_done, ignt_q.io.count)) { - val meta_dirty = !xact_tag_match || pending_coh_on_ignt != xact_meta.coh - when(meta_dirty) { pending_coh := pending_coh_on_ignt } - state := Mux(meta_dirty, s_meta_write, - Mux(ifin_cnt > UInt(0) || io.ignt().requiresAck(), - s_inner_finish, s_idle)) - } - } - is(s_meta_write) { - when(io.meta.write.ready) { - state := Mux(ifin_cnt > UInt(0), s_inner_finish, s_idle) - } - } - is(s_inner_finish) { - when(doneNextCycleCounter(io.inner.finish.valid, ifin_cnt)) { - state := s_idle - } - } + // Defined here because of Chisel default wire demands, used in s_meta_resp + val pending_coh_on_hit = HierarchicalMetadata( + io.meta.resp.bits.meta.coh.inner, + io.meta.resp.bits.meta.coh.outer.onHit(xact.op_code())) + + // State machine updates and transaction handler metadata intialization + when(state === s_idle && io.inner.acquire.valid) { + xact_src := io.inner.acquire.bits.header.src + xact := io.iacq() + xact.data := UInt(0) + pending_puts := Mux( + io.iacq().isBuiltInType(Acquire.putBlockType), + dropPendingBitWhenBeatHasData(io.inner.acquire), + UInt(0)) + pending_reads := Mux( + io.iacq().isBuiltInType(Acquire.getBlockType) || !io.iacq().isBuiltInType(), + SInt(-1, width = innerDataBeats), + addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire)).toUInt + pending_writes := addPendingBitWhenBeatHasData(io.inner.acquire) + pending_resps := UInt(0) + pending_ignt_data := UInt(0) + pending_meta_write := UInt(0) + state := s_meta_read + } + when(state === s_meta_read && io.meta.read.ready) { state := s_meta_resp } + when(state === s_meta_resp && io.meta.resp.valid) { + xact_tag_match := io.meta.resp.bits.tag_match + xact_old_meta := io.meta.resp.bits.meta + xact_way_en := io.meta.resp.bits.way_en + val coh = io.meta.resp.bits.meta.coh + val tag_match = io.meta.resp.bits.tag_match + val is_hit = (if(!isLastLevelCache) tag_match && coh.outer.isHit(xact.op_code()) + else xact.isBuiltInType(Acquire.putBlockType) || + tag_match && coh.outer.isValid()) + val needs_writeback = !tag_match && + xact.allocate() && + (coh.outer.requiresVoluntaryWriteback() || + coh.inner.requiresProbesOnVoluntaryWriteback()) + val needs_inner_probes = tag_match && coh.inner.requiresProbes(xact) + when(!tag_match || is_hit && pending_coh_on_hit != coh) { pending_meta_write := Bool(true) } + pending_coh := Mux(is_hit, pending_coh_on_hit, coh) + when(needs_inner_probes) { + val full_sharers = coh.inner.full() + val mask_self = Mux( + xact.requiresSelfProbe(), + coh.inner.full() | UIntToOH(xact_src), + coh.inner.full() & ~UIntToOH(xact_src)) + val mask_incoherent = mask_self & ~io.incoherent.toBits + pending_iprbs := mask_incoherent + } + state := Mux(needs_writeback, s_wb_req, + Mux(needs_inner_probes, s_inner_probe, + Mux(!is_hit, s_outer_acquire, s_busy))) + } + when(state === s_wb_req && io.wb.req.ready) { state := s_wb_resp } + when(state === s_wb_resp && io.wb.resp.valid) { + // If we're overwriting the whole block in a last level cache we can + // just do it without fetching any data from memory + val skip_outer_acquire = Bool(isLastLevelCache) && + xact.isBuiltInType(Acquire.putBlockType) + state := Mux(!skip_outer_acquire, s_outer_acquire, s_busy) + } + when(state === s_inner_probe && !(pending_iprbs.orR || pending_irels)) { + // Tag matches, so if this is the last level cache we can use the data without upgrading permissions + val skip_outer_acquire = + (if(!isLastLevelCache) xact_old_meta.coh.outer.isHit(xact.op_code()) + else xact.isBuiltInType(Acquire.putBlockType) || xact_old_meta.coh.outer.isValid()) + state := Mux(!skip_outer_acquire, s_outer_acquire, s_busy) + } + when(state === s_outer_acquire && oacq_data_done) { state := s_busy } + when(state === s_busy && all_pending_done) { state := s_meta_write } + when(state === s_meta_write && (io.meta.write.ready || !pending_meta_write)) { + wmask_buffer.foreach { w => w := UInt(0) } + state := s_idle } - // Handle Get and Put merging - when(io.inner.acquire.fire()) { - when (io.iacq().hasData()) { - val beat = io.iacq().addr_beat - val wmask = io.iacq().wmask() - val full = FillInterleaved(8, wmask) - data_buffer(beat) := (~full & data_buffer(beat)) | (full & io.iacq().data) - wmask_buffer(beat) := wmask | Mux(state === s_idle, Bits(0), wmask_buffer(beat)) - } - when(!io.iacq().hasMultibeatData()) { ignt_q.io.enq.valid := Bool(true) } - } + // These IOs are used for routing in the parent + val in_same_set = xact.addr_block(idxMSB,idxLSB) === io.iacq().addr_block(idxMSB,idxLSB) + io.has_release_match := xact.conflicts(io.irel()) && + !io.irel().isVoluntary() && + (state === s_inner_probe) + io.has_acquire_match := can_merge_iacq_put || can_merge_iacq_get + io.has_acquire_conflict := in_same_set && (state != s_idle) && !io.has_acquire_match + //TODO: relax from in_same_set to xact.conflicts(io.iacq())? + // Checks for illegal behavior assert(!(state != s_idle && io.inner.acquire.fire() && io.inner.acquire.bits.header.src != xact_src), "AcquireTracker accepted data beat from different network source than initial request.") @@ -1019,7 +976,7 @@ class L2WritebackUnitIO extends HierarchicalXactTrackerIO { class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val io = new L2WritebackUnitIO - val s_idle :: s_inner_probe :: s_data_read :: s_data_resp :: s_outer_release :: s_outer_grant :: s_outer_finish :: s_wb_resp :: Nil = Enum(UInt(), 8) + val s_idle :: s_inner_probe :: s_data_read :: s_data_resp :: s_outer_release :: s_outer_grant :: s_wb_resp :: Nil = Enum(UInt(), 7) val state = Reg(init=s_idle) val xact_addr_block = Reg(io.wb.req.bits.addr_block.clone) @@ -1027,10 +984,9 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_way_en = Reg{ Bits(width = nWays) } val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) } val xact_id = Reg{ UInt() } - val pending_ofin = Reg{ io.outer.finish.bits.clone } val irel_had_data = Reg(init = Bool(false)) - val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) + val irel_cnt = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) val curr_probe_dst = PriorityEncoder(pending_probes) val full_sharers = io.wb.req.bits.coh.inner.full() @@ -1056,15 +1012,12 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.acquire.valid := Bool(false) io.outer.probe.ready := Bool(false) io.outer.release.valid := Bool(false) // default - io.outer.release.bits.payload := xact_coh.outer.makeVoluntaryWriteback( - client_xact_id = UInt(trackerId), - addr_block = xact_addr_block, - addr_beat = orel_data_cnt, - data = data_buffer(orel_data_cnt)) - io.outer.release.bits.header.src := UInt(bankId) + io.outer.release.bits := xact_coh.outer.makeVoluntaryWriteback( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block, + addr_beat = orel_data_cnt, + data = data_buffer(orel_data_cnt)) io.outer.grant.ready := Bool(false) // default - io.outer.finish.valid := Bool(false) // default - io.outer.finish.bits := pending_ofin io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) @@ -1100,7 +1053,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val needs_inner_probes = io.wb.req.bits.coh.inner.requiresProbesOnVoluntaryWriteback() when(needs_inner_probes) { pending_probes := mask_incoherent - release_count := PopCount(mask_incoherent) + irel_cnt := PopCount(mask_incoherent) } state := Mux(needs_inner_probes, s_inner_probe, s_data_read) } @@ -1121,12 +1074,12 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { xact_coh.outer := pending_ocoh_on_irel data_buffer(io.irel().addr_beat) := io.irel().data } - // We don't decrement release_count until we've received all the data beats. + // We don't decrement irel_cnt until we've received all the data beats. when(!io.irel().hasData() || irel_data_done) { - release_count := release_count - UInt(1) + irel_cnt := irel_cnt - UInt(1) } } - when(release_count === UInt(0)) { + when(irel_cnt === UInt(0)) { state := Mux(irel_had_data, // If someone released a dirty block s_outer_release, // write that block back, otherwise Mux(xact_coh.outer.requiresVoluntaryWriteback(), @@ -1152,19 +1105,9 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { is(s_outer_grant) { io.outer.grant.ready := Bool(true) when(io.outer.grant.valid) { - when(io.ognt().requiresAck()) { - pending_ofin.payload := pending_ofin_on_ognt - pending_ofin.header.dst := io.outer.grant.bits.header.src - state := s_outer_finish - }.otherwise { - state := s_wb_resp - } + state := s_wb_resp } } - is(s_outer_finish) { - io.outer.finish.valid := Bool(true) - when(io.outer.finish.ready) { state := s_wb_resp } - } is(s_wb_resp) { io.wb.resp.valid := Bool(true) state := s_idle diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 81d95453..787847b9 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -127,21 +127,11 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { val tx_size = Mux(!nack && (cmd === cmd_readmem || cmd === cmd_readcr || cmd === cmd_writecr), size, UInt(0)) val tx_done = io.host.out.ready && tx_subword_count.andR && (tx_word_count === tx_size || tx_word_count > UInt(0) && packet_ram_raddr.andR) - val mem_acked = Reg(init=Bool(false)) - val mem_gxid = Reg(Bits()) - val mem_gsrc = Reg(UInt()) - val mem_needs_ack = Reg(Bool()) - when (io.mem.grant.valid) { - mem_acked := Bool(true) - mem_gxid := io.mem.grant.bits.payload.manager_xact_id - mem_gsrc := io.mem.grant.bits.header.src - mem_needs_ack := io.mem.grant.bits.payload.requiresAck() - } - io.mem.grant.ready := Bool(true) - - val state_rx :: state_pcr_req :: state_pcr_resp :: state_mem_rreq :: state_mem_wreq :: state_mem_rresp :: state_mem_wresp :: state_mem_finish :: state_tx :: Nil = Enum(UInt(), 9) + val state_rx :: state_pcr_req :: state_pcr_resp :: state_mem_rreq :: state_mem_wreq :: state_mem_rresp :: state_mem_wresp :: state_tx :: Nil = Enum(UInt(), 8) val state = Reg(init=state_rx) + val (cnt, cnt_done) = Counter((state === state_mem_wreq && io.mem.acquire.ready) || + (state === state_mem_rresp && io.mem.grant.valid), dataBeats) val rx_cmd = Mux(rx_word_count === UInt(0), next_cmd, cmd) when (state === state_rx && rx_done) { state := Mux(rx_cmd === cmd_readmem, state_mem_rreq, @@ -149,28 +139,18 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { Mux(rx_cmd === cmd_readcr || rx_cmd === cmd_writecr, state_pcr_req, state_tx))) } - - val (cnt, cnt_done) = Counter((state === state_mem_wreq && io.mem.acquire.ready) || - (state === state_mem_rresp && io.mem.grant.valid), dataBeats) when (state === state_mem_wreq) { when (cnt_done) { state := state_mem_wresp } } when (state === state_mem_rreq) { when(io.mem.acquire.ready) { state := state_mem_rresp } } - when (state === state_mem_wresp) { - when (mem_acked) { - state := state_mem_finish - mem_acked := Bool(false) - } + when (state === state_mem_wresp && io.mem.grant.valid) { + state := Mux(cmd === cmd_readmem || pos === UInt(1), state_tx, state_rx) + pos := pos - UInt(1) + addr := addr + UInt(1 << offsetBits-3) } - when (state === state_mem_rresp) { - when (cnt_done) { - state := state_mem_finish - mem_acked := Bool(false) - } - } - when (state === state_mem_finish && io.mem.finish.ready) { + when (state === state_mem_rresp && cnt_done) { state := Mux(cmd === cmd_readmem || pos === UInt(1), state_tx, state_rx) pos := pos - UInt(1) addr := addr + UInt(1 << offsetBits-3) @@ -187,8 +167,8 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { val mem_req_data = (0 until n).map { i => val ui = UInt(i, log2Up(n)) when (state === state_mem_rresp && io.mem.grant.valid) { - packet_ram(Cat(io.mem.grant.bits.payload.addr_beat, ui)) := - io.mem.grant.bits.payload.data((i+1)*short_request_bits-1, i*short_request_bits) + packet_ram(Cat(io.mem.grant.bits.addr_beat, ui)) := + io.mem.grant.bits.data((i+1)*short_request_bits-1, i*short_request_bits) } packet_ram(Cat(cnt, ui)) }.reverse.reduce(_##_) @@ -202,9 +182,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { client_xact_id = UInt(0), data = mem_req_data), GetBlock(addr_block = init_addr)) - io.mem.finish.valid := (state === state_mem_finish) && mem_needs_ack - io.mem.finish.bits.payload.manager_xact_id := mem_gxid - io.mem.finish.bits.header.dst := mem_gsrc + io.mem.grant.ready := Bool(true) val pcrReadData = Reg(Bits(width = io.cpu(0).pcr_rep.bits.getWidth)) for (i <- 0 until nCores) { diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 059fbdcf..878ce19a 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -207,7 +207,7 @@ class MemDesser(w: Int) extends Module // test rig side //Adapter betweewn an UncachedTileLinkIO and a mem controller MemIO -class MemIOTileLinkIOConverter(qDepth: Int) extends Module { +class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { val io = new Bundle { val tl = new TileLinkIO().flip val mem = new MemIO @@ -216,11 +216,10 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { val mifAddrBits = params(MIFAddrBits) val mifDataBits = params(MIFDataBits) val mifDataBeats = params(MIFDataBeats) - val tlDataBits = params(TLDataBits) - val tlDataBeats = params(TLDataBeats) val dataBits = tlDataBits*tlDataBeats - require(tlDataBits*tlDataBeats == mifDataBits*mifDataBeats) - require(params(TLClientXactIdBits) <= params(MIFTagBits)) + val dstIdBits = params(LNHeaderBits) + require(tlDataBits*tlDataBeats == mifDataBits*mifDataBeats, "Data sizes between LLC and MC don't agree") + require(dstIdBits + tlClientXactIdBits < mifTagBits, "MemIO converter is going truncate tags: " + dstIdBits + " + " + tlClientXactIdBits + " >= " + mifTagBits) io.tl.acquire.ready := Bool(false) io.tl.probe.valid := Bool(false) @@ -231,6 +230,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { val gnt_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), 2)) io.tl.grant <> gnt_arb.io.out + val dst_off = dstIdBits + tlClientXactIdBits val acq_has_data = io.tl.acquire.bits.payload.hasData() val rel_has_data = io.tl.release.bits.payload.hasData() @@ -253,6 +253,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { g_type = Mux(data_from_rel, Grant.voluntaryAckType, Grant.putAckType), client_xact_id = tag_out >> UInt(1), manager_xact_id = UInt(0)) + gnt_arb.io.in(1).bits.header.dst := (if(dstIdBits > 0) tag_out(dst_off, tlClientXactIdBits + 1) else UInt(0)) + gnt_arb.io.in(1).bits.header.src := UInt(0) if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) { val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth)) @@ -273,7 +275,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { when(io.tl.release.valid) { active_out := Bool(true) cmd_sent_out := Bool(false) - tag_out := Cat(io.tl.release.bits.payload.client_xact_id, + tag_out := Cat(io.tl.release.bits.header.src, + io.tl.release.bits.payload.client_xact_id, io.tl.release.bits.payload.isVoluntary()) addr_out := io.tl.release.bits.payload.addr_block has_data := rel_has_data @@ -284,7 +287,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { } .elsewhen(io.tl.acquire.valid) { active_out := Bool(true) cmd_sent_out := Bool(false) - tag_out := Cat(io.tl.acquire.bits.payload.client_xact_id, + tag_out := Cat(io.tl.release.bits.header.src, + io.tl.acquire.bits.payload.client_xact_id, io.tl.acquire.bits.payload.isBuiltInType()) addr_out := io.tl.acquire.bits.payload.addr_block has_data := acq_has_data @@ -350,7 +354,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { data_from_rel := Bool(true) make_grant_ack := Bool(true) io.mem.req_data.bits.data := io.tl.release.bits.payload.data - val tag = Cat(io.tl.release.bits.payload.client_xact_id, + val tag = Cat(io.tl.release.bits.header.src, + io.tl.release.bits.payload.client_xact_id, io.tl.release.bits.payload.isVoluntary()) val addr = io.tl.release.bits.payload.addr_block io.mem.req_cmd.bits.tag := tag @@ -364,7 +369,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { make_grant_ack := acq_has_data io.mem.req_data.bits.data := io.tl.acquire.bits.payload.data io.mem.req_cmd.bits.rw := acq_has_data - val tag = Cat(io.tl.acquire.bits.payload.client_xact_id, + val tag = Cat(io.tl.acquire.bits.header.src, + io.tl.acquire.bits.payload.client_xact_id, io.tl.acquire.bits.payload.isBuiltInType()) val addr = io.tl.acquire.bits.payload.addr_block io.mem.req_cmd.bits.tag := tag @@ -421,6 +427,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { manager_xact_id = UInt(0), addr_beat = tl_cnt_in, data = tl_buf_in(tl_cnt_in)) + gnt_arb.io.in(0).bits.header.dst := (if(dstIdBits > 0) tag_in(dst_off, tlClientXactIdBits + 1) else UInt(0)) + gnt_arb.io.in(0).bits.header.src := UInt(0) when(!active_in) { io.mem.resp.ready := Bool(true) @@ -454,6 +462,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { manager_xact_id = UInt(0), addr_beat = tl_cnt_in, data = io.mem.resp.bits.data) + gnt_arb.io.in(0).bits.header.dst := (if(dstIdBits > 0) io.mem.resp.bits.tag(dst_off, tlClientXactIdBits + 1) else UInt(0)) + gnt_arb.io.in(0).bits.header.src := UInt(0) } } diff --git a/uncore/src/main/scala/metadata.scala b/uncore/src/main/scala/metadata.scala index 8f5d7e90..af5504c8 100644 --- a/uncore/src/main/scala/metadata.scala +++ b/uncore/src/main/scala/metadata.scala @@ -18,6 +18,7 @@ class ClientMetadata extends CoherenceMetadata { val state = UInt(width = co.clientStateWidth) def ===(rhs: ClientMetadata): Bool = this.state === rhs.state + def !=(rhs: ClientMetadata): Bool = !this.===(rhs) def isValid(dummy: Int = 0): Bool = co.isValid(this) def isHit(cmd: UInt): Bool = co.isHit(cmd, this) @@ -99,6 +100,7 @@ class ManagerMetadata extends CoherenceMetadata { def ===(rhs: ManagerMetadata): Bool = //this.state === rhs.state && TODO: Fix 0-width wires in Chisel this.sharers === rhs.sharers + def !=(rhs: ManagerMetadata): Bool = !this.===(rhs) def full(dummy: Int = 0) = co.dir.full(this.sharers) def requiresProbes(acq: Acquire): Bool = co.requiresProbes(acq, this) @@ -171,7 +173,7 @@ class HierarchicalMetadata extends CoherenceMetadata { val outer: ClientMetadata = Bundle(new ClientMetadata, {case TLId => params(OuterTLId)}) def ===(rhs: HierarchicalMetadata): Bool = this.inner === rhs.inner && this.outer === rhs.outer - def !=(rhs: HierarchicalMetadata): Bool = !(this === rhs) + def !=(rhs: HierarchicalMetadata): Bool = !this.===(rhs) } object HierarchicalMetadata { diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index 161a9a0c..7fdc8e09 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -6,6 +6,7 @@ import Chisel._ case object LNManagers extends Field[Int] case object LNClients extends Field[Int] case object LNEndpoints extends Field[Int] +case object LNHeaderBits extends Field[Int] class PhysicalHeader(n: Int) extends Bundle { val src = UInt(width = log2Up(n)) @@ -47,8 +48,8 @@ class BasicCrossbar[T <: Data](n: Int, dType: T, count: Int = 1, needsLock: Opti abstract class LogicalNetwork extends Module class LogicalHeader extends Bundle { - val src = UInt(width = log2Up(params(LNEndpoints))) - val dst = UInt(width = log2Up(params(LNEndpoints))) + val src = UInt(width = params(LNHeaderBits)) + val dst = UInt(width = params(LNHeaderBits)) } class LogicalNetworkIO[T <: Data](dType: T) extends Bundle { diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 0aff9aab..c2bc2d30 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -3,7 +3,8 @@ package uncore import Chisel._ import scala.math.max -import scala.reflect.ClassTag +import scala.reflect._ +import scala.reflect.runtime.universe._ // Parameters exposed to the top-level design, set based on // external requirements or design space exploration @@ -11,17 +12,22 @@ import scala.reflect.ClassTag case object TLId extends Field[String] // Unique name per network case object TLCoherencePolicy extends Field[CoherencePolicy] case object TLBlockAddrBits extends Field[Int] -case object TLManagerXactIdBits extends Field[Int] -case object TLClientXactIdBits extends Field[Int] +case object TLMaxClientXacts extends Field[Int] +case object TLMaxClientPorts extends Field[Int] +case object TLMaxManagerXacts extends Field[Int] case object TLDataBits extends Field[Int] case object TLDataBeats extends Field[Int] case object TLNetworkIsOrderedP2P extends Field[Boolean] abstract trait TileLinkParameters extends UsesParameters { val tlBlockAddrBits = params(TLBlockAddrBits) - val tlClientXactIdBits = params(TLClientXactIdBits) - val tlManagerXactIdBits = params(TLManagerXactIdBits) + val tlMaxClientXacts = params(TLMaxClientXacts) + val tlMaxClientPorts = params(TLMaxClientPorts) + val tlMaxManagerXacts = params(TLMaxManagerXacts) + val tlClientXactIdBits = log2Up(tlMaxClientXacts*tlMaxClientPorts) + val tlManagerXactIdBits = log2Up(tlMaxManagerXacts) val tlDataBits = params(TLDataBits) + val tlDataBytes = tlDataBits/8 val tlDataBeats = params(TLDataBeats) val tlCoh = params(TLCoherencePolicy) val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits/8 @@ -41,12 +47,16 @@ abstract trait TileLinkParameters extends UsesParameters { val amoAluOperandBits = params(AmoAluOperandBits) } -abstract class TLBundle extends Bundle with TileLinkParameters +abstract class TLBundle extends Bundle with TileLinkParameters { +} abstract class TLModule extends Module with TileLinkParameters // Directionality of message channel // Used to hook up logical network ports to physical network ports -trait TileLinkChannel extends TLBundle +trait TileLinkChannel extends TLBundle { + def hasData(dummy: Int = 0): Bool + def hasMultibeatData(dummy: Int = 0): Bool +} trait ClientToManagerChannel extends TileLinkChannel trait ManagerToClientChannel extends TileLinkChannel trait ClientToClientChannel extends TileLinkChannel // Unused for now @@ -97,8 +107,7 @@ class Acquire extends ClientToManagerChannel val addrByteOff = tlMemoryOperandSizeBits + opSizeOff val addrByteMSB = tlByteAddrBits + addrByteOff def allocate(dummy: Int = 0) = union(0) - def op_code(dummy: Int = 0) = Mux(isBuiltInType() && - (a_type === Acquire.putType || a_type === Acquire.putBlockType), + def op_code(dummy: Int = 0) = Mux(isBuiltInType(Acquire.putType) || isBuiltInType(Acquire.putBlockType), M_XWR, union(opSizeOff-1, opCodeOff)) def op_size(dummy: Int = 0) = union(addrByteOff-1, opSizeOff) def addr_byte(dummy: Int = 0) = union(addrByteMSB-1, addrByteOff) @@ -328,6 +337,8 @@ class Probe extends ManagerToClientChannel val p_type = UInt(width = tlCoh.probeTypeWidth) def is(t: UInt) = p_type === t + def hasData(dummy: Int = 0) = Bool(false) + def hasMultibeatData(dummy: Int = 0) = Bool(false) } object Probe { @@ -383,6 +394,7 @@ class Grant extends ManagerToClientChannel // Helper funcs def isBuiltInType(dummy: Int = 0): Bool = is_builtin_type + def isBuiltInType(t: UInt): Bool = is_builtin_type && g_type === t def is(t: UInt):Bool = g_type === t def hasData(dummy: Int = 0): Bool = Mux(isBuiltInType(), Grant.typesWithData.contains(g_type), @@ -394,7 +406,7 @@ class Grant extends ManagerToClientChannel def isVoluntary(dummy: Int = 0): Bool = isBuiltInType() && (g_type === Grant.voluntaryAckType) def requiresAck(dummy: Int = 0): Bool = !Bool(tlNetworkPreservesPointToPointOrdering) && !isVoluntary() def makeFinish(dummy: Int = 0): Finish = { - val f = Bundle(new Finish, { case TLManagerXactIdBits => tlManagerXactIdBits }) + val f = Bundle(new Finish, { case TLMaxManagerXacts => tlMaxManagerXacts }) f.manager_xact_id := this.manager_xact_id f } @@ -428,7 +440,10 @@ object Grant { } } -class Finish extends ClientToManagerChannel with HasManagerTransactionId +class Finish extends ClientToManagerChannel with HasManagerTransactionId { + def hasData(dummy: Int = 0) = Bool(false) + def hasMultibeatData(dummy: Int = 0) = Bool(false) +} // Complete IO definitions for two types of TileLink clients class UncachedTileLinkIO extends TLBundle { @@ -455,33 +470,22 @@ class TileLinkIOWrapper extends TLModule { io.out.probe.ready := Bool(true) io.out.release.valid := Bool(false) } -object TileLinkIOWrapper { - def apply(utl: UncachedTileLinkIO, p: Parameters): TileLinkIO = { - val conv = Module(new TileLinkIOWrapper)(p) - conv.io.in <> utl - conv.io.out - } - def apply(utl: UncachedTileLinkIO): TileLinkIO = { - val conv = Module(new TileLinkIOWrapper) - conv.io.in <> utl - conv.io.out - } - def apply(tl: TileLinkIO) = tl -} // This version of TileLinkIO does not contain network headers for packets // that originate in the Clients (i.e. Acquire and Release). These headers // are provided in the top-level that instantiates the clients and network. // By eliding the header subbundles within the clients we can enable // hierarchical P&R while minimizing unconnected port errors in GDS. +// Secondly, this version of the interface elides Finish messages, with the +// assumption that a FinishUnit has been coupled to the TileLinkIO port +// to deal with acking received Grants. class HeaderlessUncachedTileLinkIO extends TLBundle { val acquire = new DecoupledIO(new Acquire) - val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip - val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) + val grant = new DecoupledIO(new Grant).flip } class HeaderlessTileLinkIO extends HeaderlessUncachedTileLinkIO { - val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip + val probe = new DecoupledIO(new Probe).flip val release = new DecoupledIO(new Release) } @@ -492,17 +496,231 @@ class HeaderlessTileLinkIOWrapper extends TLModule { } io.out.acquire <> io.in.acquire io.out.grant <> io.in.grant - io.out.finish <> io.in.finish io.out.probe.ready := Bool(true) io.out.release.valid := Bool(false) } -object HeaderlessTileLinkIOWrapper { +object TileLinkIOWrapper { + def apply(utl: HeaderlessUncachedTileLinkIO, p: Parameters): HeaderlessTileLinkIO = { + val conv = Module(new HeaderlessTileLinkIOWrapper)(p) + conv.io.in <> utl + conv.io.out + } def apply(utl: HeaderlessUncachedTileLinkIO): HeaderlessTileLinkIO = { val conv = Module(new HeaderlessTileLinkIOWrapper) conv.io.in <> utl conv.io.out } + def apply(tl: HeaderlessTileLinkIO): HeaderlessTileLinkIO = tl + def apply(utl: UncachedTileLinkIO, p: Parameters): TileLinkIO = { + val conv = Module(new TileLinkIOWrapper)(p) + conv.io.in <> utl + conv.io.out + } + def apply(utl: UncachedTileLinkIO): TileLinkIO = { + val conv = Module(new TileLinkIOWrapper) + conv.io.in <> utl + conv.io.out + } + def apply(tl: TileLinkIO): TileLinkIO = tl +} + +trait HasDataBeatCounters { + type HasBeat = TileLinkChannel with HasTileLinkBeatId + type HasClientId = TileLinkChannel with HasClientTransactionId + type LNAcquire = LogicalNetworkIO[Acquire] + type LNRelease = LogicalNetworkIO[Release] + type LNGrant = LogicalNetworkIO[Grant] + + def connectDataBeatCounter[S <: TileLinkChannel : ClassTag](inc: Bool, data: S, beat: UInt) = { + val multi = data.hasMultibeatData() + val (multi_cnt, multi_done) = Counter(inc && multi, data.tlDataBeats) + val cnt = Mux(multi, multi_cnt, beat) + val done = Mux(multi, multi_done, inc) + (cnt, done) + } + + def connectOutgoingDataBeatCounter[T <: Data : TypeTag]( + in: DecoupledIO[T], + beat: UInt = UInt(0)): (UInt, Bool) = { + in.bits match { + case p: TileLinkChannel if typeTag[T].tpe <:< typeTag[TileLinkChannel].tpe => + connectDataBeatCounter(in.fire(), p, beat) + case ln: LNGrant if typeTag[T].tpe <:< typeTag[LNGrant].tpe => + connectDataBeatCounter(in.fire(), ln.payload, beat) + case _ => { require(false, "Don't know how to connect a beat counter to " + typeTag[T].tpe); (UInt(0), Bool(false))} + } + } + + def connectIncomingDataBeatCounters[T <: HasClientId : ClassTag]( + in: DecoupledIO[LogicalNetworkIO[T]], + entries: Int): Vec[Bool] = { + val id = in.bits.payload.client_xact_id + Vec((0 until entries).map { i => + connectDataBeatCounter(in.fire() && id === UInt(i), in.bits.payload, UInt(0))._2 + }) + } + + def connectIncomingDataBeatCounter[T <: Data : TypeTag](in: DecoupledIO[T]): Bool = { + in.bits match { + case p: TileLinkChannel if typeTag[T].tpe <:< typeTag[TileLinkChannel].tpe => + connectDataBeatCounter(in.fire(), p, UInt(0))._2 + case ln: LNAcquire if typeTag[T].tpe =:= typeTag[LNAcquire].tpe => + connectDataBeatCounter(in.fire(), ln.payload, UInt(0))._2 + case ln: LNRelease if typeTag[T].tpe =:= typeTag[LNRelease].tpe => + connectDataBeatCounter(in.fire(), ln.payload, UInt(0))._2 + case ln: LNGrant if typeTag[T].tpe =:= typeTag[LNGrant].tpe => + connectDataBeatCounter(in.fire(), ln.payload, UInt(0))._2 + case _ => { require(false, "Don't know how to connect a beat counter to " + typeTag[T].tpe); Bool(false)} + } + } + + def connectHeaderlessTwoWayBeatCounter[ T <: TileLinkChannel : ClassTag, S <: TileLinkChannel : ClassTag]( + max: Int, + up: DecoupledIO[T], + down: DecoupledIO[S], + beat: UInt): (Bool, UInt, Bool, UInt, Bool) = { + val cnt = Reg(init = UInt(0, width = log2Up(max+1))) + val (up_idx, do_inc) = connectDataBeatCounter(up.fire(), up.bits, beat) + val (down_idx, do_dec) = connectDataBeatCounter(down.fire(), down.bits, beat) + cnt := Mux(do_dec, + Mux(do_inc, cnt, cnt - UInt(1)), + Mux(do_inc, cnt + UInt(1), cnt)) + (cnt > UInt(0), up_idx, do_inc, down_idx, do_dec) + } + + def connectTwoWayBeatCounter[ T <: TileLinkChannel : ClassTag, S <: TileLinkChannel : ClassTag]( + max: Int, + up: DecoupledIO[LogicalNetworkIO[T]], + down: DecoupledIO[LogicalNetworkIO[S]], + inc: T => Bool = (t: T) => Bool(true), + dec: S => Bool = (s: S) => Bool(true)): (Bool, UInt, Bool, UInt, Bool) = { + val cnt = Reg(init = UInt(0, width = log2Up(max+1))) + val (up_idx, up_done) = connectDataBeatCounter(up.fire(), up.bits.payload, UInt(0)) + val (down_idx, down_done) = connectDataBeatCounter(down.fire(), down.bits.payload, UInt(0)) + val do_inc = up_done && inc(up.bits.payload) + val do_dec = down_done && dec(down.bits.payload) + cnt := Mux(do_dec, + Mux(do_inc, cnt, cnt - UInt(1)), + Mux(do_inc, cnt + UInt(1), cnt)) + (cnt > UInt(0), up_idx, up_done, down_idx, down_done) + } +} + +class FinishQueueEntry extends TLBundle { + val fin = new Finish + val dst = UInt(width = log2Up(params(LNEndpoints))) +} + +class FinishQueue(entries: Int) extends Queue(new FinishQueueEntry, entries) + +class FinishUnit(srcId: Int = 0) extends TLModule + with HasDataBeatCounters { + val io = new Bundle { + val grant = Decoupled(new LogicalNetworkIO(new Grant)).flip + val refill = Decoupled(new Grant) + val finish = Decoupled(new LogicalNetworkIO(new Finish)) + val ready = Bool(OUTPUT) + val grant_done = Bool(OUTPUT) + val pending_finish = Bool(OUTPUT) + } + + val entries = 1 << tlClientXactIdBits + val g = io.grant.bits.payload + assert(g.client_xact_id <= UInt(entries), "No grant beat counter provisioned, only " + entries) + + val done = connectIncomingDataBeatCounters(io.grant, entries).reduce(_||_) + val q = Module(new FinishQueue(entries)) + + q.io.enq.valid := io.grant.valid && g.requiresAck() && (!g.hasMultibeatData() || done) + q.io.enq.bits.fin := g.makeFinish() + q.io.enq.bits.dst := io.grant.bits.header.src + + io.finish.bits.header.src := UInt(srcId) + io.finish.bits.header.dst := q.io.deq.bits.dst + io.finish.bits.payload := q.io.deq.bits.fin + io.finish.valid := q.io.deq.valid + q.io.deq.ready := io.finish.ready + + io.refill.valid := io.grant.valid + io.refill.bits := io.grant.bits.payload + io.grant.ready := (q.io.enq.ready || !g.requiresAck()) && (io.refill.ready || !g.hasData()) + + io.ready := q.io.enq.ready + io.grant_done := done + io.pending_finish := q.io.deq.valid +} + +object TileLinkHeaderOverwriter { + def apply[T <: ClientToManagerChannel]( + in: DecoupledIO[LogicalNetworkIO[T]], + clientId: Int, + passThrough: Boolean): DecoupledIO[LogicalNetworkIO[T]] = { + val out = in.clone.asDirectionless + out.bits.payload := in.bits.payload + out.bits.header.src := UInt(clientId) + out.bits.header.dst := (if(passThrough) in.bits.header.dst else UInt(0)) + out.valid := in.valid + in.ready := out.ready + out + } + + def apply[T <: ClientToManagerChannel with HasCacheBlockAddress]( + in: DecoupledIO[LogicalNetworkIO[T]], + clientId: Int, + nBanks: Int, + addrConvert: UInt => UInt): DecoupledIO[LogicalNetworkIO[T]] = { + val out: DecoupledIO[LogicalNetworkIO[T]] = apply(in, clientId, false) + out.bits.header.dst := addrConvert(in.bits.payload.addr_block) + out + } + + def apply[T <: ClientToManagerChannel with HasCacheBlockAddress : ClassTag]( + in: DecoupledIO[T], + clientId: Int, + addrConvert: UInt => UInt): DecoupledIO[LogicalNetworkIO[T]] = { + val out = new DecoupledIO(new LogicalNetworkIO(in.bits.clone)).asDirectionless + out.bits.payload := in.bits + out.bits.header.src := UInt(clientId) + out.bits.header.dst := addrConvert(in.bits.addr_block) + out.valid := in.valid + in.ready := out.ready + out + } +} + +class TileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) extends TLModule { + val io = new Bundle { + val client = new HeaderlessTileLinkIO().flip + val network = new TileLinkIO + } + + val finisher = Module(new FinishUnit(clientId)) + finisher.io.grant <> io.network.grant + io.network.finish <> finisher.io.finish + + val acq_with_header = TileLinkHeaderOverwriter(io.client.acquire, clientId, addrConvert) + val rel_with_header = TileLinkHeaderOverwriter(io.client.release, clientId, addrConvert) + val prb_without_header = DecoupledLogicalNetworkIOUnwrapper(io.network.probe) + val gnt_without_header = finisher.io.refill + + io.network.acquire.bits := acq_with_header.bits + io.network.acquire.valid := acq_with_header.valid && finisher.io.ready + acq_with_header.ready := io.network.acquire.ready && finisher.io.ready + io.network.release <> rel_with_header + io.client.probe <> prb_without_header + io.client.grant <> gnt_without_header +} + +object TileLinkNetworkPort { + def apply[T <: Data]( + client: HeaderlessTileLinkIO, + clientId: Int = 0, + addrConvert: UInt => UInt = u => UInt(0))(implicit p: Parameters): TileLinkIO = { + val port = Module(new TileLinkNetworkPort(clientId, addrConvert))(p) + port.io.client <> client + port.io.network + } } class TileLinkEnqueuer(depths: (Int, Int, Int, Int, Int)) extends Module { @@ -529,7 +747,6 @@ object TileLinkEnqueuer { } abstract trait HasArbiterTypes { - val arbN: Int type ManagerSourcedWithId = ManagerToClientChannel with HasClientTransactionId type ClientSourcedWithId = ClientToManagerChannel with HasClientTransactionId type ClientSourcedWithIdAndData = ClientToManagerChannel with @@ -538,20 +755,19 @@ abstract trait HasArbiterTypes { } // Utility functions for constructing TileLinkIO arbiters -abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule - with HasArbiterTypes { - +trait TileLinkArbiterLike extends HasArbiterTypes with TileLinkParameters{ + val arbN: Int // These are filled in depending on whether the arbiter mucks with the // client ids and then needs to revert them on the way back def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int): Bits def managerSourcedClientXactId(in: ManagerSourcedWithId): Bits def arbIdx(in: ManagerSourcedWithId): UInt - def hookupClientSource[M <: ClientSourcedWithIdAndData : ClassTag] - (clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], - mngr: DecoupledIO[LogicalNetworkIO[M]]) { + def hookupClientSource[M <: ClientSourcedWithIdAndData : ClassTag]( + clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], + mngr: DecoupledIO[LogicalNetworkIO[M]]) { def hasData(m: LogicalNetworkIO[M]) = m.payload.hasMultibeatData() - val arb = Module(new LockingRRArbiter(mngr.bits.clone, arbN, params(TLDataBeats), Some(hasData _))) + val arb = Module(new LockingRRArbiter(mngr.bits.clone, arbN, tlDataBeats, Some(hasData _))) clts.zipWithIndex.zip(arb.io.in).map{ case ((req, id), arb) => { arb.valid := req.valid arb.bits := req.bits @@ -561,11 +777,11 @@ abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule arb.io.out <> mngr } - def hookupClientSourceHeaderless[M <: ClientSourcedWithIdAndData : ClassTag] - (clts: Seq[DecoupledIO[M]], - mngr: DecoupledIO[M]) { + def hookupClientSourceHeaderless[M <: ClientSourcedWithIdAndData : ClassTag]( + clts: Seq[DecoupledIO[M]], + mngr: DecoupledIO[M]) { def hasData(m: M) = m.hasMultibeatData() - val arb = Module(new LockingRRArbiter(mngr.bits.clone, arbN, params(TLDataBeats), Some(hasData _))) + val arb = Module(new LockingRRArbiter(mngr.bits.clone, arbN, tlDataBeats, Some(hasData _))) clts.zipWithIndex.zip(arb.io.in).map{ case ((req, id), arb) => { arb.valid := req.valid arb.bits := req.bits @@ -575,17 +791,23 @@ abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule arb.io.out <> mngr } - def hookupFinish[M <: LogicalNetworkIO[Finish] : ClassTag] - (clts: Seq[DecoupledIO[M]], - mngr: DecoupledIO[M]) { - val arb = Module(new RRArbiter(mngr.bits.clone, arbN)) - arb.io.in zip clts map { case (arb, req) => arb <> req } - arb.io.out <> mngr + def hookupManagerSourceWithHeader[M <: ManagerToClientChannel]( + clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], + mngr: DecoupledIO[LogicalNetworkIO[M]]) { + mngr.ready := Bool(false) + for (i <- 0 until arbN) { + clts(i).valid := Bool(false) + when (mngr.bits.header.dst === UInt(i)) { + clts(i).valid := mngr.valid + mngr.ready := clts(i).ready + } + clts(i).bits := mngr.bits + } } - def hookupManagerSourceWithId[M <: ManagerSourcedWithId] - (clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], - mngr: DecoupledIO[LogicalNetworkIO[M]]) { + def hookupManagerSourceWithId[M <: ManagerSourcedWithId]( + clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], + mngr: DecoupledIO[LogicalNetworkIO[M]]) { mngr.ready := Bool(false) for (i <- 0 until arbN) { clts(i).valid := Bool(false) @@ -594,24 +816,45 @@ abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule mngr.ready := clts(i).ready } clts(i).bits := mngr.bits - clts(i).bits.payload.client_xact_id := - managerSourcedClientXactId(mngr.bits.payload) + clts(i).bits.payload.client_xact_id := managerSourcedClientXactId(mngr.bits.payload) } } - def hookupManagerSourceBroadcast[M <: ManagerToClientChannel] - (clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], - mngr: DecoupledIO[LogicalNetworkIO[M]]) { + def hookupManagerSourceHeaderlessWithId[M <: ManagerSourcedWithId]( + clts: Seq[DecoupledIO[M]], + mngr: DecoupledIO[M]) { + mngr.ready := Bool(false) + for (i <- 0 until arbN) { + clts(i).valid := Bool(false) + when (arbIdx(mngr.bits) === UInt(i)) { + clts(i).valid := mngr.valid + mngr.ready := clts(i).ready + } + clts(i).bits := mngr.bits + clts(i).bits.client_xact_id := managerSourcedClientXactId(mngr.bits) + } + } + + def hookupManagerSourceBroadcast[M <: Data]( + clts: Seq[DecoupledIO[M]], + mngr: DecoupledIO[M]) { clts.map{ _.valid := mngr.valid } clts.map{ _.bits := mngr.bits } mngr.ready := clts.map(_.ready).reduce(_&&_) } + + def hookupFinish[M <: LogicalNetworkIO[Finish] : ClassTag]( + clts: Seq[DecoupledIO[M]], + mngr: DecoupledIO[M]) { + val arb = Module(new RRArbiter(mngr.bits.clone, arbN)) + arb.io.in zip clts map { case (arb, req) => arb <> req } + arb.io.out <> mngr + } } -abstract class UncachedTileLinkIOArbiter(n: Int) - extends TileLinkArbiterLike(n) { +abstract class UncachedTileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiterLike { val io = new Bundle { - val in = Vec.fill(n){new UncachedTileLinkIO}.flip + val in = Vec.fill(arbN){new UncachedTileLinkIO}.flip val out = new UncachedTileLinkIO } hookupClientSource(io.in.map(_.acquire), io.out.acquire) @@ -619,9 +862,9 @@ abstract class UncachedTileLinkIOArbiter(n: Int) hookupManagerSourceWithId(io.in.map(_.grant), io.out.grant) } -abstract class TileLinkIOArbiter(n: Int) extends TileLinkArbiterLike(n) { +abstract class TileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiterLike { val io = new Bundle { - val in = Vec.fill(n){new TileLinkIO}.flip + val in = Vec.fill(arbN){new TileLinkIO}.flip val out = new TileLinkIO } hookupClientSource(io.in.map(_.acquire), io.out.acquire) @@ -631,8 +874,32 @@ abstract class TileLinkIOArbiter(n: Int) extends TileLinkArbiterLike(n) { hookupManagerSourceWithId(io.in.map(_.grant), io.out.grant) } +class HeaderlessUncachedTileLinkIOArbiter(val arbN: Int) extends Module + with TileLinkArbiterLike + with AppendsArbiterId { + val io = new Bundle { + val in = Vec.fill(arbN){new HeaderlessUncachedTileLinkIO}.flip + val out = new HeaderlessUncachedTileLinkIO + } + hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) + hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) +} + +class HeaderlessTileLinkIOArbiter(val arbN: Int) extends Module + with TileLinkArbiterLike + with AppendsArbiterId { + val io = new Bundle { + val in = Vec.fill(arbN){new HeaderlessTileLinkIO}.flip + val out = new HeaderlessTileLinkIO + } + hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) + hookupClientSourceHeaderless(io.in.map(_.release), io.out.release) + hookupManagerSourceBroadcast(io.in.map(_.probe), io.out.probe) + hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) +} + // Appends the port index of the arbiter to the client_xact_id -abstract trait AppendsArbiterId extends HasArbiterTypes { +trait AppendsArbiterId extends TileLinkArbiterLike { def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = Cat(in.client_xact_id, UInt(id, log2Up(arbN))) def managerSourcedClientXactId(in: ManagerSourcedWithId) = @@ -641,14 +908,14 @@ abstract trait AppendsArbiterId extends HasArbiterTypes { } // Uses the client_xact_id as is (assumes it has been set to port index) -abstract trait PassesId extends HasArbiterTypes { +trait PassesId extends TileLinkArbiterLike { def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = in.client_xact_id def managerSourcedClientXactId(in: ManagerSourcedWithId) = in.client_xact_id def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id } // Overwrites some default client_xact_id with the port idx -abstract trait UsesNewId extends HasArbiterTypes { +trait UsesNewId extends TileLinkArbiterLike { def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = UInt(id, log2Up(arbN)) def managerSourcedClientXactId(in: ManagerSourcedWithId) = UInt(0) def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index de55c0e2..44981459 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -2,7 +2,8 @@ package uncore import Chisel._ -import scala.reflect.ClassTag +import scala.reflect._ +import scala.reflect.runtime.universe._ case object NReleaseTransactors extends Field[Int] case object NProbeTransactors extends Field[Int] @@ -82,26 +83,24 @@ trait HasInnerTLIO extends CoherenceAgentBundle { } trait HasUncachedOuterTLIO extends CoherenceAgentBundle { - val outer = Bundle(new UncachedTileLinkIO)(outerTLParams) - def oacq(dummy: Int = 0) = outer.acquire.bits.payload - def ognt(dummy: Int = 0) = outer.grant.bits.payload - def ofin(dummy: Int = 0) = outer.finish.bits.payload + val outer = Bundle(new HeaderlessUncachedTileLinkIO)(outerTLParams) + def oacq(dummy: Int = 0) = outer.acquire.bits + def ognt(dummy: Int = 0) = outer.grant.bits } trait HasCachedOuterTLIO extends CoherenceAgentBundle { - val outer = Bundle(new TileLinkIO)(outerTLParams) - def oacq(dummy: Int = 0) = outer.acquire.bits.payload - def oprb(dummy: Int = 0) = outer.probe.bits.payload - def orel(dummy: Int = 0) = outer.release.bits.payload - def ognt(dummy: Int = 0) = outer.grant.bits.payload - def ofin(dummy: Int = 0) = outer.finish.bits.payload + val outer = Bundle(new HeaderlessTileLinkIO)(outerTLParams) + def oacq(dummy: Int = 0) = outer.acquire.bits + def oprb(dummy: Int = 0) = outer.probe.bits + def orel(dummy: Int = 0) = outer.release.bits + def ognt(dummy: Int = 0) = outer.grant.bits } class ManagerTLIO extends HasInnerTLIO with HasUncachedOuterTLIO abstract class CoherenceAgent extends CoherenceAgentModule { def innerTL: TileLinkIO - def outerTL: TileLinkIO + def outerTL: HeaderlessTileLinkIO def incoherent: Vec[Bool] } @@ -131,39 +130,47 @@ trait HasTrackerConflictIO extends Bundle { class ManagerXactTrackerIO extends ManagerTLIO with HasTrackerConflictIO class HierarchicalXactTrackerIO extends HierarchicalTLIO with HasTrackerConflictIO -abstract class XactTracker extends CoherenceAgentModule { - def connectDataBeatCounter[S <: HasTileLinkData : ClassTag](inc: Bool, data: S, beat: UInt) = { - val multi = data.hasMultibeatData() - val (multi_cnt, multi_done) = Counter(inc && multi, data.tlDataBeats) - val cnt = Mux(multi, multi_cnt, beat) - val done = Mux(multi, multi_done, inc) - (cnt, done) +abstract class XactTracker extends CoherenceAgentModule + with HasDataBeatCounters { + def addPendingBitWhenBeat[T <: HasBeat](inc: Bool, in: T): UInt = Fill(in.tlDataBeats, inc) & UIntToOH(in.addr_beat) + def dropPendingBitWhenBeat[T <: HasBeat](dec: Bool, in: T): UInt = ~Fill(in.tlDataBeats, dec) | ~UIntToOH(in.addr_beat) + + def addPendingBitWhenBeatHasData[T <: Data : TypeTag](in: DecoupledIO[T]): UInt = { + in.bits match { + case p: HasBeat if typeTag[T].tpe <:< typeTag[HasBeat].tpe => + addPendingBitWhenBeat(in.fire() && p.hasData(), p) + case ln: LNAcquire if typeTag[T].tpe <:< typeTag[LNAcquire].tpe => + addPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) + case ln: LNRelease if typeTag[T].tpe <:< typeTag[LNRelease].tpe => + addPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) + case ln: LNGrant if typeTag[T].tpe <:< typeTag[LNGrant].tpe => + addPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) + case _ => { require(false, "Don't know how track beats of " + typeTag[T].tpe); UInt(0) } + } } - def connectOutgoingDataBeatCounter[T <: HasTileLinkData : ClassTag]( - in: DecoupledIO[LogicalNetworkIO[T]], - beat: UInt = UInt(0)) = { - connectDataBeatCounter(in.fire(), in.bits.payload, beat) - } - - def connectIncomingDataBeatCounter[T <: HasTileLinkData : ClassTag](in: DecoupledIO[LogicalNetworkIO[T]]) = { - connectDataBeatCounter(in.fire(), in.bits.payload, UInt(0))._2 - } - - def addPendingBitWhenHasData[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[LogicalNetworkIO[T]]) = { - Fill(in.bits.payload.tlDataBeats, in.fire() && in.bits.payload.hasData()) & - UIntToOH(in.bits.payload.addr_beat) - } - - def dropPendingBitWhenHasData[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[LogicalNetworkIO[T]]) = { - ~Fill(in.bits.payload.tlDataBeats, in.fire() && in.bits.payload.hasData()) | - ~UIntToOH(in.bits.payload.addr_beat) - } - - def addPendingBitWhenGetOrAtomic(in: DecoupledIO[LogicalNetworkIO[Acquire]]) = { + def addPendingBitWhenBeatIsGetOrAtomic(in: DecoupledIO[LogicalNetworkIO[Acquire]]): UInt = { val a = in.bits.payload - Fill(a.tlDataBeats, in.fire() && a.isBuiltInType() && - (a.is(Acquire.getType) || a.is(Acquire.getBlockType) || a.is(Acquire.putAtomicType))) & - UIntToOH(a.addr_beat) + val isGetOrAtomic = a.isBuiltInType() && + (Vec(Acquire.getType, Acquire.getBlockType, Acquire.putAtomicType).contains(a.a_type)) + addPendingBitWhenBeat(in.fire() && isGetOrAtomic, in.bits.payload) + } + + def dropPendingBitWhenBeatHasData[T <: Data : TypeTag](in: DecoupledIO[T]): UInt = { + in.bits match { + case p: HasBeat if typeTag[T].tpe <:< typeTag[HasBeat].tpe => + dropPendingBitWhenBeat(in.fire() && p.hasData(), p) + case ln: LNAcquire if typeTag[T].tpe <:< typeTag[LNAcquire].tpe => + dropPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) + case ln: LNRelease if typeTag[T].tpe <:< typeTag[LNRelease].tpe => + dropPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) + case ln: LNGrant if typeTag[T].tpe <:< typeTag[LNGrant].tpe => + dropPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) + case _ => { require(false, "Don't know how track beats of " + typeTag[T].tpe); UInt(0) } + } + } + + def dropPendingBitAtDest(in: DecoupledIO[LogicalNetworkIO[Probe]]): UInt = { + ~Fill(nCoherentClients, in.fire()) | ~UIntToOH(in.bits.header.dst) } } diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util.scala index 14575aa5..f02cf2c3 100644 --- a/uncore/src/main/scala/util.scala +++ b/uncore/src/main/scala/util.scala @@ -34,15 +34,15 @@ object ZCounter { } } -class FlowThroughSerializer[T <: HasTileLinkData](gen: LogicalNetworkIO[T], n: Int) extends Module { +class FlowThroughSerializer[T <: HasTileLinkData](gen: T, n: Int) extends Module { val io = new Bundle { val in = Decoupled(gen.clone).flip val out = Decoupled(gen.clone) val cnt = UInt(OUTPUT, log2Up(n)) val done = Bool(OUTPUT) } - val narrowWidth = io.in.bits.payload.data.getWidth / n - require(io.in.bits.payload.data.getWidth % narrowWidth == 0) + val narrowWidth = io.in.bits.data.getWidth / n + require(io.in.bits.data.getWidth % narrowWidth == 0) if(n == 1) { io.in <> io.out @@ -51,12 +51,12 @@ class FlowThroughSerializer[T <: HasTileLinkData](gen: LogicalNetworkIO[T], n: I } else { val cnt = Reg(init=UInt(0, width = log2Up(n))) val wrap = cnt === UInt(n-1) - val rbits = Reg(init=io.in.bits) + val rbits = Reg(io.in.bits.clone) val active = Reg(init=Bool(false)) val shifter = Vec.fill(n){Bits(width = narrowWidth)} (0 until n).foreach { - i => shifter(i) := rbits.payload.data((i+1)*narrowWidth-1,i*narrowWidth) + i => shifter(i) := rbits.data((i+1)*narrowWidth-1,i*narrowWidth) } io.done := Bool(false) @@ -65,16 +65,16 @@ class FlowThroughSerializer[T <: HasTileLinkData](gen: LogicalNetworkIO[T], n: I io.out.valid := active || io.in.valid io.out.bits := io.in.bits when(!active && io.in.valid) { - when(io.in.bits.payload.hasData()) { + when(io.in.bits.hasData()) { cnt := Mux(io.out.ready, UInt(1), UInt(0)) rbits := io.in.bits active := Bool(true) } - io.done := !io.in.bits.payload.hasData() + io.done := !io.in.bits.hasData() } when(active) { io.out.bits := rbits - io.out.bits.payload.data := shifter(cnt) + io.out.bits.data := shifter(cnt) when(io.out.ready) { cnt := cnt + UInt(1) when(wrap) { @@ -86,3 +86,13 @@ class FlowThroughSerializer[T <: HasTileLinkData](gen: LogicalNetworkIO[T], n: I } } } + +object FlowThroughSerializer { + def apply[T <: HasTileLinkData](in: DecoupledIO[T], n: Int): DecoupledIO[T] = { + val fs = Module(new FlowThroughSerializer(in.bits, n)) + fs.io.in.valid := in.valid + fs.io.in.bits := in.bits + in.ready := fs.io.in.ready + fs.io.out + } +} From 3cf1778c928a717e96d863330762dca14e9161a0 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 3 Apr 2015 17:24:44 -0700 Subject: [PATCH 364/374] moved ecc lib to uncore, l2 checks for partial write masks when ecc is enabled --- uncore/src/main/scala/cache.scala | 15 ++- uncore/src/main/scala/ecc.scala | 146 +++++++++++++++++++++++++++++ uncore/src/main/scala/uncore.scala | 6 +- uncore/src/main/scala/util.scala | 8 ++ 4 files changed, 171 insertions(+), 4 deletions(-) create mode 100644 uncore/src/main/scala/ecc.scala diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 05588203..492ecc0e 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -15,6 +15,7 @@ case object NPrimaryMisses extends Field[Int] case object NSecondaryMisses extends Field[Int] case object CacheBlockBytes extends Field[Int] case object CacheBlockOffsetBits extends Field[Int] +case object ECCCode extends Field[Option[Code]] abstract trait CacheParameters extends UsesParameters { val nSets = params(NSets) @@ -28,6 +29,7 @@ abstract trait CacheParameters extends UsesParameters { val rowBits = params(RowBits) val rowBytes = rowBits/8 val rowOffBits = log2Up(rowBytes) + val code = params(ECCCode).getOrElse(new IdentityCode) } abstract class CacheBundle extends Bundle with CacheParameters @@ -176,6 +178,7 @@ abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgen require(rowBits == innerDataBits) // TODO: relax this by improving s_data_* states val nSecondaryMisses = params(NSecondaryMisses) val isLastLevelCache = true + val ignoresWriteMask = !params(ECCCode).isEmpty } abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters @@ -462,6 +465,12 @@ abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { def dropPendingBitInternal[T <: HasL2BeatAddr] (in: ValidIO[T]) = ~Fill(in.bits.refillCycles, in.valid) | ~UIntToOH(in.bits.addr_beat) + + def addPendingBitWhenBeatHasPartialWritemask(in: DecoupledIO[LogicalNetworkIO[Acquire]]): UInt = { + val a = in.bits.payload + val isPartial = a.wmask() != Acquire.fullWriteMask + addPendingBitWhenBeat(in.fire() && isPartial && Bool(ignoresWriteMask), in.bits.payload) + } } class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTracker { @@ -805,7 +814,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { dropPendingBit(io.data.read) & dropPendingBitWhenBeatHasData(io.inner.release) & dropPendingBitWhenBeatHasData(io.outer.grant)) | - addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire) + addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire) | + addPendingBitWhenBeatHasPartialWritemask(io.inner.acquire) val curr_read_beat = PriorityEncoder(pending_reads) io.data.read.valid := state === s_busy && pending_reads.orR && @@ -880,7 +890,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_reads := Mux( io.iacq().isBuiltInType(Acquire.getBlockType) || !io.iacq().isBuiltInType(), SInt(-1, width = innerDataBeats), - addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire)).toUInt + (addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire) | + addPendingBitWhenBeatHasPartialWritemask(io.inner.acquire)).toUInt) pending_writes := addPendingBitWhenBeatHasData(io.inner.acquire) pending_resps := UInt(0) pending_ignt_data := UInt(0) diff --git a/uncore/src/main/scala/ecc.scala b/uncore/src/main/scala/ecc.scala new file mode 100644 index 00000000..b5864b2c --- /dev/null +++ b/uncore/src/main/scala/ecc.scala @@ -0,0 +1,146 @@ +// See LICENSE for license details. + +package uncore + +import Chisel._ + +abstract class Decoding +{ + def uncorrected: Bits + def corrected: Bits + def correctable: Bool + def uncorrectable: Bool + def error = correctable || uncorrectable +} + +abstract class Code +{ + def width(w0: Int): Int + def encode(x: Bits): Bits + def decode(x: Bits): Decoding +} + +class IdentityCode extends Code +{ + def width(w0: Int) = w0 + def encode(x: Bits) = x + def decode(y: Bits) = new Decoding { + def uncorrected = y + def corrected = y + def correctable = Bool(false) + def uncorrectable = Bool(false) + } +} + +class ParityCode extends Code +{ + def width(w0: Int) = w0+1 + def encode(x: Bits) = Cat(x.xorR, x) + def decode(y: Bits) = new Decoding { + def uncorrected = y(y.getWidth-2,0) + def corrected = uncorrected + def correctable = Bool(false) + def uncorrectable = y.xorR + } +} + +class SECCode extends Code +{ + def width(k: Int) = { + val m = new Unsigned(k).log2 + 1 + k + m + (if((1 << m) < m+k+1) 1 else 0) + } + def encode(x: Bits) = { + val k = x.getWidth + require(k > 0) + val n = width(k) + + val y = for (i <- 1 to n) yield { + if (isPow2(i)) { + val r = for (j <- 1 to n; if j != i && (j & i) != 0) + yield x(mapping(j)) + r reduce (_^_) + } else + x(mapping(i)) + } + Vec(y).toBits + } + def decode(y: Bits) = new Decoding { + val n = y.getWidth + require(n > 0 && !isPow2(n)) + + val p2 = for (i <- 0 until log2Up(n)) yield 1 << i + val syndrome = p2 map { i => + val r = for (j <- 1 to n; if (j & i) != 0) + yield y(j-1) + r reduce (_^_) + } + val s = Vec(syndrome).toBits + + private def swizzle(z: Bits) = Vec((1 to n).filter(i => !isPow2(i)).map(i => z(i-1))).toBits + def uncorrected = swizzle(y) + def corrected = swizzle(((y.toUInt << UInt(1)) ^ UIntToOH(s)) >> UInt(1)) + def correctable = s.orR + def uncorrectable = Bool(false) + } + private def mapping(i: Int) = i-1-log2Up(i) +} + +class SECDEDCode extends Code +{ + private val sec = new SECCode + private val par = new ParityCode + + def width(k: Int) = sec.width(k)+1 + def encode(x: Bits) = par.encode(sec.encode(x)) + def decode(x: Bits) = new Decoding { + val secdec = sec.decode(x(x.getWidth-2,0)) + val pardec = par.decode(x) + + def uncorrected = secdec.uncorrected + def corrected = secdec.corrected + def correctable = pardec.uncorrectable + def uncorrectable = !pardec.uncorrectable && secdec.correctable + } +} + +object ErrGen +{ + // generate a 1-bit error with approximate probability 2^-f + def apply(width: Int, f: Int): Bits = { + require(width > 0 && f >= 0 && log2Up(width) + f <= 16) + UIntToOH(LFSR16()(log2Up(width)+f-1,0))(width-1,0) + } + def apply(x: Bits, f: Int): Bits = x ^ apply(x.getWidth, f) +} + +class SECDEDTest extends Module +{ + val code = new SECDEDCode + val k = 4 + val n = code.width(k) + + val io = new Bundle { + val original = Bits(OUTPUT, k) + val encoded = Bits(OUTPUT, n) + val injected = Bits(OUTPUT, n) + val uncorrected = Bits(OUTPUT, k) + val corrected = Bits(OUTPUT, k) + val correctable = Bool(OUTPUT) + val uncorrectable = Bool(OUTPUT) + } + + val c = Counter(Bool(true), 1 << k) + val numErrors = Counter(c._2, 3)._1 + val e = code.encode(c._1) + val i = e ^ Mux(numErrors < UInt(1), UInt(0), ErrGen(n, 1)) ^ Mux(numErrors < UInt(2), UInt(0), ErrGen(n, 1)) + val d = code.decode(i) + + io.original := c._1 + io.encoded := e + io.injected := i + io.uncorrected := d.uncorrected + io.corrected := d.corrected + io.correctable := d.correctable + io.uncorrectable := d.uncorrectable +} diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 44981459..62a53129 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -132,8 +132,10 @@ class HierarchicalXactTrackerIO extends HierarchicalTLIO with HasTrackerConflict abstract class XactTracker extends CoherenceAgentModule with HasDataBeatCounters { - def addPendingBitWhenBeat[T <: HasBeat](inc: Bool, in: T): UInt = Fill(in.tlDataBeats, inc) & UIntToOH(in.addr_beat) - def dropPendingBitWhenBeat[T <: HasBeat](dec: Bool, in: T): UInt = ~Fill(in.tlDataBeats, dec) | ~UIntToOH(in.addr_beat) + def addPendingBitWhenBeat[T <: HasBeat](inc: Bool, in: T): UInt = + Fill(in.tlDataBeats, inc) & UIntToOH(in.addr_beat) + def dropPendingBitWhenBeat[T <: HasBeat](dec: Bool, in: T): UInt = + ~Fill(in.tlDataBeats, dec) | ~UIntToOH(in.addr_beat) def addPendingBitWhenBeatHasData[T <: Data : TypeTag](in: DecoupledIO[T]): UInt = { in.bits match { diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util.scala index f02cf2c3..65c5d6cd 100644 --- a/uncore/src/main/scala/util.scala +++ b/uncore/src/main/scala/util.scala @@ -5,6 +5,14 @@ package uncore import Chisel._ import scala.math._ +class Unsigned(x: Int) { + require(x >= 0) + def clog2: Int = { require(x > 0); ceil(log(x)/log(2)).toInt } + def log2: Int = { require(x > 0); floor(log(x)/log(2)).toInt } + def isPow2: Boolean = x > 0 && (x & (x-1)) == 0 + def nextPow2: Int = if (x == 0) 1 else 1 << clog2 +} + object MuxBundle { def apply[T <: Data] (default: T, mapping: Seq[(Bool, T)]): T = { mapping.reverse.foldLeft(default)((b, a) => Mux(a._1, a._2, b)) From 90f800d87dd1d05b3837f710d9dd5d131747149e Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 13 Apr 2015 15:57:06 -0700 Subject: [PATCH 365/374] Grant bugfixes and more comments --- uncore/src/main/scala/cache.scala | 25 +++++++++++++++++-------- uncore/src/main/scala/tilelink.scala | 3 ++- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 492ecc0e..e399eb98 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -577,7 +577,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack } } is(s_inner_grant) { - io.inner.grant.valid := Bool(true) + io.inner.grant.valid := !collect_irel_data when(io.inner.grant.ready) { state := Mux(io.ignt().requiresAck(), s_inner_finish, s_idle) } @@ -587,6 +587,10 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack when(io.inner.finish.valid) { state := s_idle } } } + + // Checks for illegal behavior + assert(!(state === s_meta_resp && io.meta.resp.valid && !io.meta.resp.bits.tag_match), + "VoluntaryReleaseTracker accepted Release for a block not resident in this cache!") } @@ -748,8 +752,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { updatePendingCohWhen(io.inner.release.fire(), pending_coh_on_irel) mergeDataInner(io.inner.release) - // The following subtrascation handles misses or coherence permission - // upgrades by initiating a new transaction in the outer memory: + // Handle misses or coherence permission upgrades by initiating a new transaction in the outer memory: // // If we're allocating in this cache, we can use the current metadata // to make an appropriate custom Acquire, otherwise we copy over the @@ -804,12 +807,17 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_coh.inner.onGrant( outgoing = io.ignt(), dst = io.inner.grant.bits.header.dst), - pending_coh.outer) + Mux(ognt_data_done, + pending_coh_on_ognt.outer, + pending_coh.outer)) updatePendingCohWhen(io.inner.grant.fire(), pending_coh_on_ignt) // We must wait for as many Finishes as we sent Grants io.inner.finish.ready := state === s_busy + // We read from the the cache at this level if data wasn't written back or refilled. + // We may merge Gets, requiring further beats to be read. + // If ECC requires a full writemask, we'll read out data on partial writes as well. pending_reads := (pending_reads & dropPendingBit(io.data.read) & dropPendingBitWhenBeatHasData(io.inner.release) & @@ -829,6 +837,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { addPendingBitInternal(io.data.read) mergeDataInternal(io.data.resp) + // We write data to the cache at this level if it was Put here with allocate flag, + // written back dirty, or refilled from outer memory. pending_writes := (pending_writes & dropPendingBit(io.data.write)) | addPendingBitWhenBeatHasData(io.inner.acquire) | addPendingBitWhenBeatHasData(io.inner.release) | @@ -883,11 +893,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { xact_src := io.inner.acquire.bits.header.src xact := io.iacq() xact.data := UInt(0) - pending_puts := Mux( + pending_puts := Mux( // Make sure to collect all data from a PutBlock io.iacq().isBuiltInType(Acquire.putBlockType), dropPendingBitWhenBeatHasData(io.inner.acquire), UInt(0)) - pending_reads := Mux( + pending_reads := Mux( // GetBlocks and custom types read all beats io.iacq().isBuiltInType(Acquire.getBlockType) || !io.iacq().isBuiltInType(), SInt(-1, width = innerDataBeats), (addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire) | @@ -932,8 +942,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(state === s_wb_resp && io.wb.resp.valid) { // If we're overwriting the whole block in a last level cache we can // just do it without fetching any data from memory - val skip_outer_acquire = Bool(isLastLevelCache) && - xact.isBuiltInType(Acquire.putBlockType) + val skip_outer_acquire = Bool(isLastLevelCache) && xact.isBuiltInType(Acquire.putBlockType) state := Mux(!skip_outer_acquire, s_outer_acquire, s_busy) } when(state === s_inner_probe && !(pending_iprbs.orR || pending_irels)) { diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index c2bc2d30..d639f791 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -583,6 +583,7 @@ trait HasDataBeatCounters { val cnt = Reg(init = UInt(0, width = log2Up(max+1))) val (up_idx, do_inc) = connectDataBeatCounter(up.fire(), up.bits, beat) val (down_idx, do_dec) = connectDataBeatCounter(down.fire(), down.bits, beat) + //Module.assert(!(do_dec && cnt === UInt(0)), "Decrementing 2way beat counter before ever incrementing") cnt := Mux(do_dec, Mux(do_inc, cnt, cnt - UInt(1)), Mux(do_inc, cnt + UInt(1), cnt)) @@ -632,7 +633,7 @@ class FinishUnit(srcId: Int = 0) extends TLModule val done = connectIncomingDataBeatCounters(io.grant, entries).reduce(_||_) val q = Module(new FinishQueue(entries)) - q.io.enq.valid := io.grant.valid && g.requiresAck() && (!g.hasMultibeatData() || done) + q.io.enq.valid := io.grant.fire() && g.requiresAck() && (!g.hasMultibeatData() || done) q.io.enq.bits.fin := g.makeFinish() q.io.enq.bits.dst := io.grant.bits.header.src From ce3271aef24329a2bde0e5fe8f5b60e73adac564 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 13 Apr 2015 19:00:40 -0700 Subject: [PATCH 366/374] refactor LNClients and LNManagers --- uncore/src/main/scala/broadcast.scala | 10 ++-- uncore/src/main/scala/cache.scala | 52 ++++++++++------- uncore/src/main/scala/memserdes.scala | 6 +- uncore/src/main/scala/network.scala | 2 - uncore/src/main/scala/tilelink.scala | 82 ++++++++++++++++----------- uncore/src/main/scala/uncore.scala | 33 ++--------- 6 files changed, 97 insertions(+), 88 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 98c8f0af..340db34d 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -187,7 +187,7 @@ class BroadcastVoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends Broa is(s_outer) { io.outer.acquire.valid := !collect_irel_data || irel_data_valid(oacq_data_cnt) when(oacq_data_done) { - state := Mux(xact.requiresAck(), s_grant, s_idle) + state := s_grant // converted irel to oacq, so expect grant TODO: Mux(xact.requiresAck(), s_grant, s_idle) ? } } is(s_grant) { // Forward the Grant.voluntaryAck @@ -221,13 +221,13 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact Acquire.prefetchType).contains(xact.a_type)), "Broadcast Hub does not support PutAtomics, subblock Gets/Puts, or prefetches") // TODO - val release_count = Reg(init=UInt(0, width = log2Up(nCoherentClients+1))) - val pending_probes = Reg(init=Bits(0, width = nCoherentClients)) + val release_count = Reg(init=UInt(0, width = log2Up(io.inner.tlNCoherentClients+1))) + val pending_probes = Reg(init=Bits(0, width = io.inner.tlNCoherentClients)) val curr_p_id = PriorityEncoder(pending_probes) val full_sharers = coh.full() val probe_self = io.inner.acquire.bits.payload.requiresSelfProbe() - val mask_self_true = UInt(UInt(1) << io.inner.acquire.bits.header.src, width = nCoherentClients) - val mask_self_false = ~UInt(UInt(1) << io.inner.acquire.bits.header.src, width = nCoherentClients) + val mask_self_true = UInt(UInt(1) << io.inner.acquire.bits.header.src, width = io.inner.tlNCoherentClients) + val mask_self_false = ~UInt(UInt(1) << io.inner.acquire.bits.header.src, width = io.inner.tlNCoherentClients) val mask_self = Mux(probe_self, full_sharers | mask_self_true, full_sharers & mask_self_false) val mask_incoherent = mask_self & ~io.incoherent.toBits diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index e399eb98..b193c5a7 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -182,7 +182,21 @@ abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgen } abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters -abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters +abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters { + def doInternalOutputArbitration[T <: Data : ClassTag]( + out: DecoupledIO[T], + ins: Seq[DecoupledIO[T]]) { + val arb = Module(new RRArbiter(out.bits.clone, ins.size)) + out <> arb.io.out + arb.io.in zip ins map { case (a, in) => a <> in } + } + + def doInternalInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { + val idx = in.bits.id + outs.map(_.bits := in.bits) + outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && idx === UInt(i) } + } +} trait HasL2Id extends Bundle with CoherenceAgentParameters { val id = UInt(width = log2Up(nTransactors + 1)) @@ -361,8 +375,8 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule // WritebackUnit evicts data from L2, including invalidating L1s val wb = Module(new L2WritebackUnit(nTransactors, bankId)) - doOutputArbitration(wb.io.wb.req, trackerList.map(_.io.wb.req)) - doInputRouting(wb.io.wb.resp, trackerList.map(_.io.wb.resp)) + doInternalOutputArbitration(wb.io.wb.req, trackerList.map(_.io.wb.req)) + doInternalInputRouting(wb.io.wb.resp, trackerList.map(_.io.wb.resp)) // Propagate incoherence flags (trackerList.map(_.io.incoherent) :+ wb.io.incoherent).map( _ := io.incoherent.toBits) @@ -409,13 +423,13 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule outerList zip outer_arb.io.in map { case(out, arb) => out <> arb } io.outer <> outer_arb.io.out - // Wire local memories - doOutputArbitration(io.meta.read, trackerList.map(_.io.meta.read)) - doOutputArbitration(io.meta.write, trackerList.map(_.io.meta.write)) - doOutputArbitration(io.data.read, trackerList.map(_.io.data.read) :+ wb.io.data.read) - doOutputArbitration(io.data.write, trackerList.map(_.io.data.write)) - doInputRouting(io.meta.resp, trackerList.map(_.io.meta.resp)) - doInputRouting(io.data.resp, trackerList.map(_.io.data.resp) :+ wb.io.data.resp) + // Wire local memory arrays + doInternalOutputArbitration(io.meta.read, trackerList.map(_.io.meta.read)) + doInternalOutputArbitration(io.meta.write, trackerList.map(_.io.meta.write)) + doInternalOutputArbitration(io.data.read, trackerList.map(_.io.data.read) :+ wb.io.data.read) + doInternalOutputArbitration(io.data.write, trackerList.map(_.io.data.write)) + doInternalInputRouting(io.meta.resp, trackerList.map(_.io.meta.resp)) + doInternalInputRouting(io.data.resp, trackerList.map(_.io.data.resp) :+ wb.io.data.resp) } @@ -620,19 +634,19 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) val pending_irels = - connectTwoWayBeatCounter(nCoherentClients, io.inner.probe, io.inner.release)._1 + connectTwoWayBeatCounter(io.inner.tlNCoherentClients, io.inner.probe, io.inner.release)._1 val (pending_ognts, oacq_data_idx, oacq_data_done, ognt_data_idx, ognt_data_done) = connectHeaderlessTwoWayBeatCounter(1, io.outer.acquire, io.outer.grant, xact.addr_beat) val (ignt_data_idx, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) val pending_ifins = connectTwoWayBeatCounter(nSecondaryMisses, io.inner.grant, io.inner.finish, (g: Grant) => g.requiresAck())._1 - val pending_puts = Reg(init=Bits(0, width = innerDataBeats)) - val pending_iprbs = Reg(init = Bits(0, width = nCoherentClients)) - val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) - val pending_writes = Reg(init=Bits(0, width = innerDataBeats)) - val pending_resps = Reg(init=Bits(0, width = innerDataBeats)) - val pending_ignt_data = Reg(init=Bits(0, width = innerDataBeats)) + val pending_puts = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_iprbs = Reg(init = Bits(0, width = io.inner.tlNCoherentClients)) + val pending_reads = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_writes = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_resps = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_ignt_data = Reg(init=Bits(0, width = io.inner.tlDataBeats)) val pending_meta_write = Reg{ Bool() } val all_pending_done = @@ -1006,8 +1020,8 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_id = Reg{ UInt() } val irel_had_data = Reg(init = Bool(false)) - val irel_cnt = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) - val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) + val irel_cnt = Reg(init = UInt(0, width = log2Up(io.inner.tlNCoherentClients+1))) + val pending_probes = Reg(init = Bits(0, width = io.inner.tlNCoherentClients)) val curr_probe_dst = PriorityEncoder(pending_probes) val full_sharers = io.wb.req.bits.coh.inner.full() val mask_incoherent = full_sharers & ~io.incoherent.toBits diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 878ce19a..942874cf 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -281,7 +281,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { addr_out := io.tl.release.bits.payload.addr_block has_data := rel_has_data data_from_rel := Bool(true) - make_grant_ack := Bool(true) + make_grant_ack := io.tl.release.bits.payload.requiresAck() tl_done_out := tl_wrap_out tl_buf_out(tl_cnt_out) := io.tl.release.bits.payload.data } .elsewhen(io.tl.acquire.valid) { @@ -352,7 +352,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { tl_done_out := tl_wrap_out when(io.tl.release.valid) { data_from_rel := Bool(true) - make_grant_ack := Bool(true) + make_grant_ack := io.tl.release.bits.payload.requiresAck() io.mem.req_data.bits.data := io.tl.release.bits.payload.data val tag = Cat(io.tl.release.bits.header.src, io.tl.release.bits.payload.client_xact_id, @@ -366,7 +366,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { has_data := rel_has_data } .elsewhen(io.tl.acquire.valid) { data_from_rel := Bool(false) - make_grant_ack := acq_has_data + make_grant_ack := acq_has_data // i.e. is it a Put io.mem.req_data.bits.data := io.tl.acquire.bits.payload.data io.mem.req_cmd.bits.rw := acq_has_data val tag = Cat(io.tl.acquire.bits.header.src, diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index 7fdc8e09..f61cec6c 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -3,8 +3,6 @@ package uncore import Chisel._ -case object LNManagers extends Field[Int] -case object LNClients extends Field[Int] case object LNEndpoints extends Field[Int] case object LNHeaderBits extends Field[Int] diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index d639f791..1ed95df6 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -11,25 +11,33 @@ import scala.reflect.runtime.universe._ // case object TLId extends Field[String] // Unique name per network case object TLCoherencePolicy extends Field[CoherencePolicy] -case object TLBlockAddrBits extends Field[Int] +case object TLNManagers extends Field[Int] +case object TLNClients extends Field[Int] +case object TLNCoherentClients extends Field[Int] +case object TLNIncoherentClients extends Field[Int] case object TLMaxClientXacts extends Field[Int] case object TLMaxClientPorts extends Field[Int] case object TLMaxManagerXacts extends Field[Int] +case object TLBlockAddrBits extends Field[Int] case object TLDataBits extends Field[Int] case object TLDataBeats extends Field[Int] case object TLNetworkIsOrderedP2P extends Field[Boolean] abstract trait TileLinkParameters extends UsesParameters { - val tlBlockAddrBits = params(TLBlockAddrBits) + val tlCoh = params(TLCoherencePolicy) + val tlNManagers = params(TLNManagers) + val tlNClients = params(TLNClients) + val tlNCoherentClients = params(TLNCoherentClients) + val tlNIncoherentClients = params(TLNIncoherentClients) val tlMaxClientXacts = params(TLMaxClientXacts) val tlMaxClientPorts = params(TLMaxClientPorts) val tlMaxManagerXacts = params(TLMaxManagerXacts) val tlClientXactIdBits = log2Up(tlMaxClientXacts*tlMaxClientPorts) val tlManagerXactIdBits = log2Up(tlMaxManagerXacts) + val tlBlockAddrBits = params(TLBlockAddrBits) val tlDataBits = params(TLDataBits) val tlDataBytes = tlDataBits/8 val tlDataBeats = params(TLDataBeats) - val tlCoh = params(TLCoherencePolicy) val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits/8 val tlBeatAddrBits = log2Up(tlDataBeats) val tlByteAddrBits = log2Up(tlWriteMaskBits) @@ -44,11 +52,11 @@ abstract trait TileLinkParameters extends UsesParameters { val tlGrantTypeBits = max(log2Up(Grant.nBuiltInTypes), tlCoh.grantTypeWidth) + 1 val tlNetworkPreservesPointToPointOrdering = params(TLNetworkIsOrderedP2P) + val tlNetworkDoesNotInterleaveBeats = true val amoAluOperandBits = params(AmoAluOperandBits) } -abstract class TLBundle extends Bundle with TileLinkParameters { -} +abstract class TLBundle extends Bundle with TileLinkParameters abstract class TLModule extends Module with TileLinkParameters // Directionality of message channel @@ -62,7 +70,7 @@ trait ManagerToClientChannel extends TileLinkChannel trait ClientToClientChannel extends TileLinkChannel // Unused for now // Common signals that are used in multiple channels. -// These traits are useful for type parameterization. +// These traits are useful for type parameterizing bundle wiring functions. // trait HasCacheBlockAddress extends TLBundle { val addr_block = UInt(width = tlBlockAddrBits) @@ -554,10 +562,10 @@ trait HasDataBeatCounters { def connectIncomingDataBeatCounters[T <: HasClientId : ClassTag]( in: DecoupledIO[LogicalNetworkIO[T]], - entries: Int): Vec[Bool] = { - val id = in.bits.payload.client_xact_id + entries: Int, + getId: LogicalNetworkIO[T] => UInt): Vec[Bool] = { Vec((0 until entries).map { i => - connectDataBeatCounter(in.fire() && id === UInt(i), in.bits.payload, UInt(0))._2 + connectDataBeatCounter(in.fire() && getId(in.bits) === UInt(i), in.bits.payload, UInt(0))._2 }) } @@ -615,41 +623,51 @@ class FinishQueueEntry extends TLBundle { class FinishQueue(entries: Int) extends Queue(new FinishQueueEntry, entries) -class FinishUnit(srcId: Int = 0) extends TLModule +class FinishUnit(srcId: Int = 0, outstanding: Int = 2) extends TLModule with HasDataBeatCounters { val io = new Bundle { val grant = Decoupled(new LogicalNetworkIO(new Grant)).flip val refill = Decoupled(new Grant) val finish = Decoupled(new LogicalNetworkIO(new Finish)) val ready = Bool(OUTPUT) - val grant_done = Bool(OUTPUT) - val pending_finish = Bool(OUTPUT) } - val entries = 1 << tlClientXactIdBits val g = io.grant.bits.payload - assert(g.client_xact_id <= UInt(entries), "No grant beat counter provisioned, only " + entries) - val done = connectIncomingDataBeatCounters(io.grant, entries).reduce(_||_) - val q = Module(new FinishQueue(entries)) + if(tlNetworkPreservesPointToPointOrdering) { + io.finish.valid := Bool(false) + io.refill.valid := io.grant.valid + io.refill.bits := g + io.grant.ready := io.refill.ready + io.ready := Bool(true) + } else { + // We only want to send Finishes after we have collected all beats of + // a multibeat Grant. But Grants from multiple managers or transactions may + // get interleaved, so we could need a counter for each. + val done = if(tlNetworkDoesNotInterleaveBeats) { + connectIncomingDataBeatCounter(io.grant) + } else { + val entries = 1 << tlClientXactIdBits + def getId(g: LogicalNetworkIO[Grant]) = g.payload.client_xact_id + assert(getId(io.grant.bits) <= UInt(entries), "Not enough grant beat counters, only " + entries + " entries.") + connectIncomingDataBeatCounters(io.grant, entries, getId).reduce(_||_) + } + val q = Module(new FinishQueue(outstanding)) + q.io.enq.valid := io.grant.fire() && g.requiresAck() && (!g.hasMultibeatData() || done) + q.io.enq.bits.fin := g.makeFinish() + q.io.enq.bits.dst := io.grant.bits.header.src - q.io.enq.valid := io.grant.fire() && g.requiresAck() && (!g.hasMultibeatData() || done) - q.io.enq.bits.fin := g.makeFinish() - q.io.enq.bits.dst := io.grant.bits.header.src + io.finish.bits.header.src := UInt(srcId) + io.finish.bits.header.dst := q.io.deq.bits.dst + io.finish.bits.payload := q.io.deq.bits.fin + io.finish.valid := q.io.deq.valid + q.io.deq.ready := io.finish.ready - io.finish.bits.header.src := UInt(srcId) - io.finish.bits.header.dst := q.io.deq.bits.dst - io.finish.bits.payload := q.io.deq.bits.fin - io.finish.valid := q.io.deq.valid - q.io.deq.ready := io.finish.ready - - io.refill.valid := io.grant.valid - io.refill.bits := io.grant.bits.payload - io.grant.ready := (q.io.enq.ready || !g.requiresAck()) && (io.refill.ready || !g.hasData()) - - io.ready := q.io.enq.ready - io.grant_done := done - io.pending_finish := q.io.deq.valid + io.refill.valid := io.grant.valid + io.refill.bits := g + io.grant.ready := (q.io.enq.ready || !g.requiresAck()) && io.refill.ready + io.ready := q.io.enq.ready + } } object TileLinkHeaderOverwriter { diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 62a53129..9d1c93b7 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -16,9 +16,6 @@ trait CoherenceAgentParameters extends UsesParameters { val nReleaseTransactors = 1 val nAcquireTransactors = params(NAcquireTransactors) val nTransactors = nReleaseTransactors + nAcquireTransactors - val nCoherentClients = params(NCoherentClients) - val nIncoherentClients = params(NIncoherentClients) - val nClients = nCoherentClients + nIncoherentClients def outerTLParams = params.alterPartial({ case TLId => params(OuterTLId)}) val outerDataBeats = outerTLParams(TLDataBeats) val outerDataBits = outerTLParams(TLDataBits) @@ -35,33 +32,15 @@ abstract class CoherenceAgentBundle extends Bundle with CoherenceAgentParameters abstract class CoherenceAgentModule extends Module with CoherenceAgentParameters trait HasCoherenceAgentWiringHelpers { - def doOutputArbitration[T <: Data : ClassTag]( - out: DecoupledIO[T], - ins: Seq[DecoupledIO[T]]) { - val arb = Module(new RRArbiter(out.bits.clone, ins.size)) - out <> arb.io.out - arb.io.in zip ins map { case (a, in) => a <> in } - } - - def doOutputArbitration[T <: HasTileLinkData : ClassTag, S <: LogicalNetworkIO[T] : ClassTag]( - out: DecoupledIO[S], - ins: Seq[DecoupledIO[S]]) { + def doOutputArbitration[T <: TileLinkChannel : ClassTag]( + out: DecoupledIO[LogicalNetworkIO[T]], + ins: Seq[DecoupledIO[LogicalNetworkIO[T]]]) { def lock(o: LogicalNetworkIO[T]) = o.payload.hasMultibeatData() - val arb = Module(new LockingRRArbiter( - out.bits.clone, - ins.size, - out.bits.payload.tlDataBeats, - lock _)) + val arb = Module(new LockingRRArbiter( out.bits.clone, ins.size, out.bits.payload.tlDataBeats, lock _)) out <> arb.io.out arb.io.in zip ins map { case (a, in) => a <> in } } - def doInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { - val idx = in.bits.id - outs.map(_.bits := in.bits) - outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && idx === UInt(i) } - } - def doInputRouting[T <: HasManagerTransactionId]( in: DecoupledIO[LogicalNetworkIO[T]], outs: Seq[DecoupledIO[LogicalNetworkIO[T]]]) { @@ -74,7 +53,7 @@ trait HasCoherenceAgentWiringHelpers { trait HasInnerTLIO extends CoherenceAgentBundle { val inner = Bundle(new TileLinkIO)(innerTLParams).flip - val incoherent = Vec.fill(nCoherentClients){Bool()}.asInput + val incoherent = Vec.fill(inner.tlNCoherentClients){Bool()}.asInput def iacq(dummy: Int = 0) = inner.acquire.bits.payload def iprb(dummy: Int = 0) = inner.probe.bits.payload def irel(dummy: Int = 0) = inner.release.bits.payload @@ -173,6 +152,6 @@ abstract class XactTracker extends CoherenceAgentModule } def dropPendingBitAtDest(in: DecoupledIO[LogicalNetworkIO[Probe]]): UInt = { - ~Fill(nCoherentClients, in.fire()) | ~UIntToOH(in.bits.header.dst) + ~Fill(in.bits.payload.tlNCoherentClients, in.fire()) | ~UIntToOH(in.bits.header.dst) } } From ba7a8b175207e6712efddd7c4683e052cb8691c3 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 17 Apr 2015 16:55:20 -0700 Subject: [PATCH 367/374] TileLink refactor; TileLinkPorts now available. L2Banks no longer have unique ids (suitable for hierarhical P&R). --- uncore/src/main/scala/broadcast.scala | 54 ++-- uncore/src/main/scala/cache.scala | 132 ++++---- uncore/src/main/scala/htif.scala | 2 +- uncore/src/main/scala/memserdes.scala | 185 ++++++----- uncore/src/main/scala/metadata.scala | 41 ++- uncore/src/main/scala/tilelink.scala | 427 ++++++++++++++------------ uncore/src/main/scala/uncore.scala | 84 ++--- 7 files changed, 460 insertions(+), 465 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 340db34d..339a7540 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -25,16 +25,16 @@ object DataQueueLocation { } } -class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent +class L2BroadcastHub extends ManagerCoherenceAgent with BroadcastHubParameters { val internalDataBits = new DataQueueLocation().getWidth val inStoreQueue :: inVolWBQueue :: inClientReleaseQueue :: Nil = Enum(UInt(), nDataQueueLocations) // Create SHRs for outstanding transactions - val trackerList = (0 until nReleaseTransactors).map(id => - Module(new BroadcastVoluntaryReleaseTracker(id, bankId), {case TLDataBits => internalDataBits})) ++ - (nReleaseTransactors until nTransactors).map(id => - Module(new BroadcastAcquireTracker(id, bankId), {case TLDataBits => internalDataBits})) + val trackerList = (0 until nReleaseTransactors).map(id => + Module(new BroadcastVoluntaryReleaseTracker(id), {case TLDataBits => internalDataBits})) ++ + (nReleaseTransactors until nTransactors).map(id => + Module(new BroadcastAcquireTracker(id), {case TLDataBits => internalDataBits})) // Propagate incoherence flags trackerList.map(_.io.incoherent := io.incoherent.toBits) @@ -61,7 +61,7 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent trackerAcquireIOs.zipWithIndex.foreach { case(tracker, i) => tracker.bits := io.inner.acquire.bits - tracker.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits + tracker.bits.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits tracker.valid := io.inner.acquire.valid && !block_acquires && (acquire_idx === UInt(i)) } @@ -82,7 +82,7 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent case(tracker, i) => tracker.valid := io.inner.release.valid && (release_idx === UInt(i)) tracker.bits := io.inner.release.bits - tracker.bits.payload.data := DataQueueLocation(rel_data_cnt, + tracker.bits.data := DataQueueLocation(rel_data_cnt, (if(i < nReleaseTransactors) inVolWBQueue else inClientReleaseQueue)).toBits } @@ -91,17 +91,17 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent // Wire probe requests and grant reply to clients, finish acks from clients // Note that we bypass the Grant data subbundles - io.inner.grant.bits.payload.data := io.outer.grant.bits.data - io.inner.grant.bits.payload.addr_beat := io.outer.grant.bits.addr_beat + io.inner.grant.bits.data := io.outer.grant.bits.data + io.inner.grant.bits.addr_beat := io.outer.grant.bits.addr_beat doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish)) // Create an arbiter for the one memory port - val outer_arb = Module(new HeaderlessUncachedTileLinkIOArbiter(trackerList.size), + val outer_arb = Module(new ClientUncachedTileLinkIOArbiter(trackerList.size), { case TLId => params(OuterTLId) case TLDataBits => internalDataBits }) - outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } + outer_arb.io.in <> trackerList.map(_.io.outer) // Get the pending data out of the store data queue val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.data) val is_in_sdq = outer_data_ptr.loc === inStoreQueue @@ -124,12 +124,11 @@ class BroadcastXactTracker extends XactTracker { val io = new ManagerXactTrackerIO } -class BroadcastVoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends BroadcastXactTracker { +class BroadcastVoluntaryReleaseTracker(trackerId: Int) extends BroadcastXactTracker { val s_idle :: s_outer :: s_grant :: s_ack :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) - val xact_src = Reg(io.inner.release.bits.header.src.clone) - val xact = Reg(Bundle(new Release, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val xact = Reg(Bundle(new ReleaseFromSrc, { case TLId => params(InnerTLId); case TLDataBits => 0 })) val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) } val coh = ManagerMetadata.onReset @@ -150,9 +149,7 @@ class BroadcastVoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends Broa io.inner.grant.valid := Bool(false) io.inner.finish.ready := Bool(false) - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := coh.makeGrant(xact, UInt(trackerId)) + io.inner.grant.bits := coh.makeGrant(xact, UInt(trackerId)) //TODO: Use io.outer.release instead? io.outer.acquire.bits := Bundle( @@ -175,7 +172,6 @@ class BroadcastVoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends Broa is(s_idle) { io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { - xact_src := io.inner.release.bits.header.src xact := io.irel() data_buffer(UInt(0)) := io.irel().data collect_irel_data := io.irel().hasMultibeatData() @@ -207,12 +203,11 @@ class BroadcastVoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends Broa } } -class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXactTracker { +class BroadcastAcquireTracker(trackerId: Int) extends BroadcastXactTracker { val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_mem_resp :: s_ack :: Nil = Enum(UInt(), 7) val state = Reg(init=s_idle) - val xact_src = Reg(io.inner.acquire.bits.header.src.clone) - val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val xact = Reg(Bundle(new AcquireFromSrc, { case TLId => params(InnerTLId); case TLDataBits => 0 })) val data_buffer = Vec.fill(innerDataBeats){ Reg(io.iacq().data.clone) } val coh = ManagerMetadata.onReset @@ -225,9 +220,9 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact val pending_probes = Reg(init=Bits(0, width = io.inner.tlNCoherentClients)) val curr_p_id = PriorityEncoder(pending_probes) val full_sharers = coh.full() - val probe_self = io.inner.acquire.bits.payload.requiresSelfProbe() - val mask_self_true = UInt(UInt(1) << io.inner.acquire.bits.header.src, width = io.inner.tlNCoherentClients) - val mask_self_false = ~UInt(UInt(1) << io.inner.acquire.bits.header.src, width = io.inner.tlNCoherentClients) + val probe_self = io.inner.acquire.bits.requiresSelfProbe() + val mask_self_true = UInt(UInt(1) << io.inner.acquire.bits.client_id, width = io.inner.tlNCoherentClients) + val mask_self_false = ~UInt(UInt(1) << io.inner.acquire.bits.client_id, width = io.inner.tlNCoherentClients) val mask_self = Mux(probe_self, full_sharers | mask_self_true, full_sharers & mask_self_false) val mask_incoherent = mask_self & ~io.incoherent.toBits @@ -272,21 +267,17 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact io.outer.grant.ready := Bool(false) io.inner.probe.valid := Bool(false) - io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := coh.makeProbe(xact) + io.inner.probe.bits := coh.makeProbe(curr_p_id, xact) io.inner.grant.valid := Bool(false) - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := coh.makeGrant(xact, UInt(trackerId)) // Data bypassed in parent + io.inner.grant.bits := coh.makeGrant(xact, UInt(trackerId)) // Data bypassed in parent io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) io.inner.finish.ready := Bool(false) assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && - io.inner.acquire.bits.header.src != xact_src), + io.iacq().client_id != xact.client_id), "AcquireTracker accepted data beat from different network source than initial request.") assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && @@ -316,7 +307,6 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact is(s_idle) { io.inner.acquire.ready := Bool(true) when(io.inner.acquire.valid) { - xact_src := io.inner.acquire.bits.header.src xact := io.iacq() data_buffer(UInt(0)) := io.iacq().data collect_iacq_data := io.iacq().hasMultibeatData() diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index b193c5a7..8ae4d00b 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -188,7 +188,7 @@ abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters { ins: Seq[DecoupledIO[T]]) { val arb = Module(new RRArbiter(out.bits.clone, ins.size)) out <> arb.io.out - arb.io.in zip ins map { case (a, in) => a <> in } + arb.io.in <> ins } def doInternalInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { @@ -342,14 +342,14 @@ class L2SecondaryMissInfo extends TLBundle with HasTileLinkBeatId with HasClientTransactionId -class L2HellaCacheBank(bankId: Int) extends HierarchicalCoherenceAgent +class L2HellaCacheBank extends HierarchicalCoherenceAgent with L2HellaCacheParameters { require(isPow2(nSets)) require(isPow2(nWays)) val meta = Module(new L2MetadataArray) // TODO: add delay knob val data = Module(new L2DataArray(1)) - val tshrfile = Module(new TSHRFile(bankId)) + val tshrfile = Module(new TSHRFile) tshrfile.io.inner <> io.inner io.outer <> tshrfile.io.outer io.incoherent <> tshrfile.io.incoherent @@ -362,24 +362,21 @@ class TSHRFileIO extends HierarchicalTLIO { val data = new L2DataRWIO } -class TSHRFile(bankId: Int) extends L2HellaCacheModule +class TSHRFile extends L2HellaCacheModule with HasCoherenceAgentWiringHelpers { val io = new TSHRFileIO // Create TSHRs for outstanding transactions - val trackerList = (0 until nReleaseTransactors).map { id => - Module(new L2VoluntaryReleaseTracker(id, bankId)) - } ++ (nReleaseTransactors until nTransactors).map { id => - Module(new L2AcquireTracker(id, bankId)) - } + val trackerList = (0 until nReleaseTransactors).map(id => Module(new L2VoluntaryReleaseTracker(id))) ++ + (nReleaseTransactors until nTransactors).map(id => Module(new L2AcquireTracker(id))) // WritebackUnit evicts data from L2, including invalidating L1s - val wb = Module(new L2WritebackUnit(nTransactors, bankId)) + val wb = Module(new L2WritebackUnit(nTransactors)) doInternalOutputArbitration(wb.io.wb.req, trackerList.map(_.io.wb.req)) doInternalInputRouting(wb.io.wb.resp, trackerList.map(_.io.wb.resp)) // Propagate incoherence flags - (trackerList.map(_.io.incoherent) :+ wb.io.incoherent).map( _ := io.incoherent.toBits) + (trackerList.map(_.io.incoherent) :+ wb.io.incoherent) foreach { _ := io.incoherent.toBits } // Handle acquire transaction initiation val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) @@ -419,8 +416,8 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule // Create an arbiter for the one memory port val outerList = trackerList.map(_.io.outer) :+ wb.io.outer - val outer_arb = Module(new HeaderlessTileLinkIOArbiter(outerList.size))(outerTLParams) - outerList zip outer_arb.io.in map { case(out, arb) => out <> arb } + val outer_arb = Module(new ClientTileLinkIOArbiter(outerList.size))(outerTLParams) + outer_arb.io.in <> outerList io.outer <> outer_arb.io.out // Wire local memory arrays @@ -480,21 +477,20 @@ abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { def dropPendingBitInternal[T <: HasL2BeatAddr] (in: ValidIO[T]) = ~Fill(in.bits.refillCycles, in.valid) | ~UIntToOH(in.bits.addr_beat) - def addPendingBitWhenBeatHasPartialWritemask(in: DecoupledIO[LogicalNetworkIO[Acquire]]): UInt = { - val a = in.bits.payload + def addPendingBitWhenBeatHasPartialWritemask(in: DecoupledIO[AcquireFromSrc]): UInt = { + val a = in.bits val isPartial = a.wmask() != Acquire.fullWriteMask - addPendingBitWhenBeat(in.fire() && isPartial && Bool(ignoresWriteMask), in.bits.payload) + addPendingBitWhenBeat(in.fire() && isPartial && Bool(ignoresWriteMask), a) } } -class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTracker { +class L2VoluntaryReleaseTracker(trackerId: Int) extends L2XactTracker { val io = new L2XactTrackerIO val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_inner_grant :: s_inner_finish :: Nil = Enum(UInt(), 7) val state = Reg(init=s_idle) - val xact_src = Reg(io.inner.release.bits.header.src.clone) - val xact = Reg(Bundle(new Release, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val xact = Reg(Bundle(new ReleaseFromSrc, { case TLId => params(InnerTLId); case TLDataBits => 0 })) val xact_tag_match = Reg{ Bool() } val xact_old_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } @@ -520,9 +516,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack io.inner.grant.valid := Bool(false) io.inner.finish.ready := Bool(false) - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := coh.inner.makeGrant(xact, UInt(trackerId)) + io.inner.grant.bits := coh.inner.makeGrant(xact, UInt(trackerId)) io.data.read.valid := Bool(false) io.data.write.valid := Bool(false) @@ -541,7 +535,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en io.meta.write.bits.data.tag := xact.addr_block >> UInt(idxBits) - io.meta.write.bits.data.coh.inner := xact_old_meta.coh.inner.onRelease(xact, xact_src) + io.meta.write.bits.data.coh.inner := xact_old_meta.coh.inner.onRelease(xact) io.meta.write.bits.data.coh.outer := xact_old_meta.coh.outer.onHit(M_XWR) // WB is a write io.wb.req.valid := Bool(false) @@ -558,7 +552,6 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack is(s_idle) { io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { - xact_src := io.inner.release.bits.header.src xact := io.irel() data_buffer(io.irel().addr_beat) := io.irel().data collect_irel_data := io.irel().hasMultibeatData() @@ -608,15 +601,14 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack } -class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { +class L2AcquireTracker(trackerId: Int) extends L2XactTracker { val io = new L2XactTrackerIO val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_inner_probe :: s_outer_acquire :: s_busy :: s_meta_write :: Nil = Enum(UInt(), 9) val state = Reg(init=s_idle) // State holding transaction metadata - val xact_src = Reg(io.inner.acquire.bits.header.src.clone) - val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId) })) + val xact = Reg(Bundle(new AcquireFromSrc, { case TLId => params(InnerTLId) })) val data_buffer = Vec.fill(innerDataBeats){ Reg(init=UInt(0, width = innerDataBits)) } val wmask_buffer = Vec.fill(innerDataBeats){ Reg(init=UInt(0,width = innerDataBits/8)) } val xact_tag_match = Reg{ Bool() } @@ -631,16 +623,23 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { // TODO add ignt.dst <- iacq.src // State holding progress made on processing this transaction - val iacq_data_done = - connectIncomingDataBeatCounter(io.inner.acquire) - val pending_irels = - connectTwoWayBeatCounter(io.inner.tlNCoherentClients, io.inner.probe, io.inner.release)._1 + val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) + val pending_irels = connectTwoWayBeatCounter( + max = io.inner.tlNCoherentClients, + up = io.inner.probe, + down = io.inner.release)._1 val (pending_ognts, oacq_data_idx, oacq_data_done, ognt_data_idx, ognt_data_done) = - connectHeaderlessTwoWayBeatCounter(1, io.outer.acquire, io.outer.grant, xact.addr_beat) - val (ignt_data_idx, ignt_data_done) = - connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) - val pending_ifins = - connectTwoWayBeatCounter(nSecondaryMisses, io.inner.grant, io.inner.finish, (g: Grant) => g.requiresAck())._1 + connectTwoWayBeatCounter( + max = 1, + up = io.outer.acquire, + down = io.outer.grant, + beat = xact.addr_beat) + val (ignt_data_idx, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) + val pending_ifins = connectTwoWayBeatCounter( + max = nSecondaryMisses, + up = io.inner.grant, + down = io.inner.finish, + track = (g: Grant) => g.requiresAck())._1 val pending_puts = Reg(init=Bits(0, width = io.inner.tlDataBeats)) val pending_iprbs = Reg(init = Bits(0, width = io.inner.tlNCoherentClients)) val pending_reads = Reg(init=Bits(0, width = io.inner.tlDataBeats)) @@ -695,9 +694,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { def mergeDataInternal[T <: HasL2Data with HasL2BeatAddr](in: ValidIO[T]) { when(in.valid) { mergeData(rowBits)(in.bits.addr_beat, in.bits.data) } } - def mergeDataInner[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[LogicalNetworkIO[T]]) { - when(in.fire() && in.bits.payload.hasData()) { - mergeData(innerDataBits)(in.bits.payload.addr_beat, in.bits.payload.data) + def mergeDataInner[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[T]) { + when(in.fire() && in.bits.hasData()) { + mergeData(innerDataBits)(in.bits.addr_beat, in.bits.data) } } def mergeDataOuter[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[T]) { @@ -713,7 +712,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { // and Puts-under-Put from the same client val can_merge_iacq_get = (xact.isBuiltInType(Acquire.getType) && io.iacq().isBuiltInType(Acquire.getType)) && - xact_src === io.inner.acquire.bits.header.src && //TODO + xact.client_id === io.iacq().client_id && //TODO remove xact.conflicts(io.iacq()) && state != s_idle && state != s_meta_write && !all_pending_done && @@ -728,7 +727,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.iacq().isBuiltInType(Acquire.putType)) || (xact.isBuiltInType(Acquire.putBlockType) && io.iacq().isBuiltInType(Acquire.putBlockType))) && - xact_src === io.inner.acquire.bits.header.src && //TODO + xact.client_id === io.iacq().client_id && //TODO remove xact.conflicts(io.iacq()) && state != s_idle && state != s_meta_write && !all_pending_done && @@ -749,17 +748,13 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_iprbs := pending_iprbs & dropPendingBitAtDest(io.inner.probe) val curr_probe_dst = PriorityEncoder(pending_iprbs) io.inner.probe.valid := state === s_inner_probe && pending_iprbs.orR - io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_probe_dst - io.inner.probe.bits.payload := pending_coh.inner.makeProbe(xact) + io.inner.probe.bits := pending_coh.inner.makeProbe(curr_probe_dst, xact) // Handle incoming releases from clients, which may reduce sharer counts // and/or write back dirty data io.inner.release.ready := state === s_inner_probe val pending_coh_on_irel = HierarchicalMetadata( - pending_coh.inner.onRelease( // Drop sharer - incoming = io.irel(), - src = io.inner.release.bits.header.src), + pending_coh.inner.onRelease(io.irel()), // Drop sharer Mux(io.irel().hasData(), // Dirty writeback pending_coh.outer.onHit(M_XWR), pending_coh.outer)) @@ -804,23 +799,18 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { ignt_q.io.deq.valid && (!io.ignt().hasData() || pending_ignt_data(ignt_data_idx)) - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := xact_src // TODO: ignt_q.io.deq.bits.src - io.inner.grant.bits.payload := pending_coh.inner.makeGrant( - acq = xact, - manager_xact_id = UInt(trackerId), - addr_beat = ignt_data_idx, - data = Mux(xact.is(Acquire.putAtomicType), - amo_result, - data_buffer(ignt_data_idx))) - // TODO: improve the ManagerMetadata.makeGrant to deal with possibility of - // multiple client transaction ids from merged secondary misses - io.ignt().client_xact_id := ignt_q.io.deq.bits.client_xact_id + io.inner.grant.bits := pending_coh.inner.makeGrant( + dst = xact.client_id, // TODO: ignt_q.io.deq.bits.src + acq = xact, + client_xact_id = ignt_q.io.deq.bits.client_xact_id, + manager_xact_id = UInt(trackerId), + addr_beat = ignt_data_idx, + data = Mux(xact.is(Acquire.putAtomicType), + amo_result, + data_buffer(ignt_data_idx))) val pending_coh_on_ignt = HierarchicalMetadata( - pending_coh.inner.onGrant( - outgoing = io.ignt(), - dst = io.inner.grant.bits.header.dst), + pending_coh.inner.onGrant(io.ignt()), Mux(ognt_data_done, pending_coh_on_ognt.outer, pending_coh.outer)) @@ -904,7 +894,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { // State machine updates and transaction handler metadata intialization when(state === s_idle && io.inner.acquire.valid) { - xact_src := io.inner.acquire.bits.header.src xact := io.iacq() xact.data := UInt(0) pending_puts := Mux( // Make sure to collect all data from a PutBlock @@ -943,8 +932,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val full_sharers = coh.inner.full() val mask_self = Mux( xact.requiresSelfProbe(), - coh.inner.full() | UIntToOH(xact_src), - coh.inner.full() & ~UIntToOH(xact_src)) + coh.inner.full() | UIntToOH(xact.client_id), + coh.inner.full() & ~UIntToOH(xact.client_id)) val mask_incoherent = mask_self & ~io.incoherent.toBits pending_iprbs := mask_incoherent } @@ -984,7 +973,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { // Checks for illegal behavior assert(!(state != s_idle && io.inner.acquire.fire() && - io.inner.acquire.bits.header.src != xact_src), + io.inner.acquire.bits.client_id != xact.client_id), "AcquireTracker accepted data beat from different network source than initial request.") } @@ -1007,7 +996,7 @@ class L2WritebackUnitIO extends HierarchicalXactTrackerIO { val data = new L2DataRWIO } -class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { +class L2WritebackUnit(trackerId: Int) extends L2XactTracker { val io = new L2WritebackUnitIO val s_idle :: s_inner_probe :: s_data_read :: s_data_resp :: s_outer_release :: s_outer_grant :: s_wb_resp :: Nil = Enum(UInt(), 7) @@ -1031,9 +1020,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val (read_data_cnt, read_data_done) = connectInternalDataBeatCounter(io.data.read) val resp_data_done = connectInternalDataBeatCounter(io.data.resp) - val pending_icoh_on_irel = xact_coh.inner.onRelease( - incoming = io.irel(), - src = io.inner.release.bits.header.src) + val pending_icoh_on_irel = xact_coh.inner.onRelease(io.irel()) val pending_ocoh_on_irel = xact_coh.outer.onHit(M_XWR) // WB is a write val pending_ofin_on_ognt = io.ognt().makeFinish() @@ -1054,10 +1041,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.grant.ready := Bool(false) // default io.inner.probe.valid := Bool(false) - io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_probe_dst - io.inner.probe.bits.payload := - xact_coh.inner.makeProbeForVoluntaryWriteback(xact_addr_block) + io.inner.probe.bits := xact_coh.inner.makeProbeForVoluntaryWriteback(curr_probe_dst, xact_addr_block) io.inner.grant.valid := Bool(false) io.inner.acquire.ready := Bool(false) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 787847b9..8a79cda1 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -60,7 +60,7 @@ class SCRIO extends HTIFBundle { class HTIFModuleIO extends HTIFBundle { val host = new HostIO val cpu = Vec.fill(nCores){new HTIFIO}.flip - val mem = new HeaderlessUncachedTileLinkIO + val mem = new ClientUncachedTileLinkIO val scr = new SCRIO } diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 942874cf..7497e2e4 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -18,16 +18,26 @@ case object MIFDataBits extends Field[Int] case object MIFTagBits extends Field[Int] case object MIFDataBeats extends Field[Int] -trait HasMemData extends Bundle { - val data = Bits(width = params(MIFDataBits)) +trait MIFParameters extends UsesParameters { + val mifTagBits = params(MIFTagBits) + val mifAddrBits = params(MIFAddrBits) + val mifDataBits = params(MIFDataBits) + val mifDataBeats = params(MIFDataBeats) +} + +abstract class MIFBundle extends Bundle with MIFParameters +abstract class MIFModule extends Module with MIFParameters + +trait HasMemData extends MIFBundle { + val data = Bits(width = mifDataBits) } -trait HasMemAddr extends Bundle { - val addr = UInt(width = params(MIFAddrBits)) +trait HasMemAddr extends MIFBundle { + val addr = UInt(width = mifAddrBits) } -trait HasMemTag extends Bundle { - val tag = UInt(width = params(MIFTagBits)) +trait HasMemTag extends MIFBundle { + val tag = UInt(width = mifTagBits) } class MemReqCmd extends HasMemAddr with HasMemTag { @@ -56,7 +66,7 @@ class MemSerializedIO(w: Int) extends Bundle val resp = Valid(Bits(width = w)).flip } -class MemSerdes(w: Int) extends Module +class MemSerdes(w: Int) extends MIFModule { val io = new Bundle { val wide = new MemIO().flip @@ -65,7 +75,6 @@ class MemSerdes(w: Int) extends Module val abits = io.wide.req_cmd.bits.toBits.getWidth val dbits = io.wide.req_data.bits.toBits.getWidth val rbits = io.wide.resp.bits.getWidth - val dbeats = params(MIFDataBeats) val out_buf = Reg(Bits()) val in_buf = Reg(Bits()) @@ -73,7 +82,7 @@ class MemSerdes(w: Int) extends Module val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(UInt(), 5) val state = Reg(init=s_idle) val send_cnt = Reg(init=UInt(0, log2Up((max(abits, dbits)+w-1)/w))) - val data_send_cnt = Reg(init=UInt(0, log2Up(dbeats))) + val data_send_cnt = Reg(init=UInt(0, log2Up(mifDataBeats))) val adone = io.narrow.req.ready && send_cnt === UInt((abits-1)/w) val ddone = io.narrow.req.ready && send_cnt === UInt((dbits-1)/w) @@ -109,12 +118,12 @@ class MemSerdes(w: Int) extends Module } when (state === s_write_data && ddone) { data_send_cnt := data_send_cnt + UInt(1) - state := Mux(data_send_cnt === UInt(dbeats-1), s_idle, s_write_idle) + state := Mux(data_send_cnt === UInt(mifDataBeats-1), s_idle, s_write_idle) send_cnt := UInt(0) } val recv_cnt = Reg(init=UInt(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = Reg(init=UInt(0, log2Up(dbeats))) + val data_recv_cnt = Reg(init=UInt(0, log2Up(mifDataBeats))) val resp_val = Reg(init=Bool(false)) resp_val := Bool(false) @@ -143,11 +152,11 @@ class MemDesser(w: Int) extends Module // test rig side val abits = io.wide.req_cmd.bits.toBits.getWidth val dbits = io.wide.req_data.bits.toBits.getWidth val rbits = io.wide.resp.bits.getWidth - val dbeats = params(MIFDataBeats) + val mifDataBeats = params(MIFDataBeats) require(dbits >= abits && rbits >= dbits) val recv_cnt = Reg(init=UInt(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = Reg(init=UInt(0, log2Up(dbeats))) + val data_recv_cnt = Reg(init=UInt(0, log2Up(mifDataBeats))) val adone = io.narrow.req.valid && recv_cnt === UInt((abits-1)/w) val ddone = io.narrow.req.valid && recv_cnt === UInt((dbits-1)/w) val rdone = io.narrow.resp.valid && recv_cnt === UInt((rbits-1)/w) @@ -175,13 +184,13 @@ class MemDesser(w: Int) extends Module // test rig side } when (state === s_data && io.wide.req_data.ready) { state := s_data_recv - when (data_recv_cnt === UInt(dbeats-1)) { + when (data_recv_cnt === UInt(mifDataBeats-1)) { state := s_cmd_recv } data_recv_cnt := data_recv_cnt + UInt(1) } when (rdone) { // state === s_reply - when (data_recv_cnt === UInt(dbeats-1)) { + when (data_recv_cnt === UInt(mifDataBeats-1)) { state := s_cmd_recv } recv_cnt := UInt(0) @@ -195,7 +204,7 @@ class MemDesser(w: Int) extends Module // test rig side io.wide.req_data.valid := state === s_data io.wide.req_data.bits.data := in_buf >> UInt(((rbits+w-1)/w - (dbits+w-1)/w)*w) - val dataq = Module(new Queue(new MemResp, dbeats)) + val dataq = Module(new Queue(new MemResp, mifDataBeats)) dataq.io.enq <> io.wide.resp dataq.io.deq.ready := recv_cnt === UInt((rbits-1)/w) @@ -203,19 +212,12 @@ class MemDesser(w: Int) extends Module // test rig side io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UInt(w)) } -//Adapter between a TileLinkIO and a UncachedTileLinkIO, merges voluntary - - //Adapter betweewn an UncachedTileLinkIO and a mem controller MemIO -class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { +class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule with MIFParameters { val io = new Bundle { - val tl = new TileLinkIO().flip + val tl = new ManagerTileLinkIO val mem = new MemIO } - val mifTagBits = params(MIFTagBits) - val mifAddrBits = params(MIFAddrBits) - val mifDataBits = params(MIFDataBits) - val mifDataBeats = params(MIFDataBeats) val dataBits = tlDataBits*tlDataBeats val dstIdBits = params(LNHeaderBits) require(tlDataBits*tlDataBeats == mifDataBits*mifDataBeats, "Data sizes between LLC and MC don't agree") @@ -227,12 +229,12 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { io.tl.finish.ready := Bool(true) io.mem.resp.ready := Bool(false) - val gnt_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), 2)) + val gnt_arb = Module(new Arbiter(new GrantToDst, 2)) io.tl.grant <> gnt_arb.io.out val dst_off = dstIdBits + tlClientXactIdBits - val acq_has_data = io.tl.acquire.bits.payload.hasData() - val rel_has_data = io.tl.release.bits.payload.hasData() + val acq_has_data = io.tl.acquire.bits.hasData() + val rel_has_data = io.tl.release.bits.hasData() // Decompose outgoing TL Acquires into MemIO cmd and data val active_out = Reg(init=Bool(false)) @@ -248,13 +250,12 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { val make_grant_ack = Reg(init=Bool(false)) gnt_arb.io.in(1).valid := Bool(false) - gnt_arb.io.in(1).bits.payload := Grant( - is_builtin_type = Bool(true), - g_type = Mux(data_from_rel, Grant.voluntaryAckType, Grant.putAckType), - client_xact_id = tag_out >> UInt(1), - manager_xact_id = UInt(0)) - gnt_arb.io.in(1).bits.header.dst := (if(dstIdBits > 0) tag_out(dst_off, tlClientXactIdBits + 1) else UInt(0)) - gnt_arb.io.in(1).bits.header.src := UInt(0) + gnt_arb.io.in(1).bits := Grant( + dst = (if(dstIdBits > 0) tag_out(dst_off, tlClientXactIdBits + 1) else UInt(0)), + is_builtin_type = Bool(true), + g_type = Mux(data_from_rel, Grant.voluntaryAckType, Grant.putAckType), + client_xact_id = tag_out >> UInt(1), + manager_xact_id = UInt(0)) if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) { val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth)) @@ -263,7 +264,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { mem_data_q.io.enq.valid := Bool(false) val (mif_cnt_out, mif_wrap_out) = Counter(mem_data_q.io.enq.fire(), mifDataBeats) val mif_done_out = Reg(init=Bool(false)) - val tl_buf_out = Vec.fill(tlDataBeats){ Reg(io.tl.acquire.bits.payload.data.clone) } + val tl_buf_out = Vec.fill(tlDataBeats){ Reg(io.tl.acquire.bits.data.clone) } val mif_buf_out = Vec.fill(mifDataBeats){ new MemData } mif_buf_out := mif_buf_out.fromBits(tl_buf_out.toBits) val mif_prog_out = (mif_cnt_out+UInt(1, width = log2Up(mifDataBeats+1)))*UInt(mifDataBits) @@ -275,27 +276,27 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { when(io.tl.release.valid) { active_out := Bool(true) cmd_sent_out := Bool(false) - tag_out := Cat(io.tl.release.bits.header.src, - io.tl.release.bits.payload.client_xact_id, - io.tl.release.bits.payload.isVoluntary()) - addr_out := io.tl.release.bits.payload.addr_block + tag_out := Cat(io.tl.release.bits.client_id, + io.tl.release.bits.client_xact_id, + io.tl.release.bits.isVoluntary()) + addr_out := io.tl.release.bits.addr_block has_data := rel_has_data data_from_rel := Bool(true) - make_grant_ack := io.tl.release.bits.payload.requiresAck() + make_grant_ack := io.tl.release.bits.requiresAck() tl_done_out := tl_wrap_out - tl_buf_out(tl_cnt_out) := io.tl.release.bits.payload.data + tl_buf_out(tl_cnt_out) := io.tl.release.bits.data } .elsewhen(io.tl.acquire.valid) { active_out := Bool(true) cmd_sent_out := Bool(false) - tag_out := Cat(io.tl.release.bits.header.src, - io.tl.acquire.bits.payload.client_xact_id, - io.tl.acquire.bits.payload.isBuiltInType()) - addr_out := io.tl.acquire.bits.payload.addr_block + tag_out := Cat(io.tl.release.bits.client_id, + io.tl.acquire.bits.client_xact_id, + io.tl.acquire.bits.isBuiltInType()) + addr_out := io.tl.acquire.bits.addr_block has_data := acq_has_data data_from_rel := Bool(false) make_grant_ack := acq_has_data tl_done_out := tl_wrap_out - tl_buf_out(tl_cnt_out) := io.tl.acquire.bits.payload.data + tl_buf_out(tl_cnt_out) := io.tl.acquire.bits.data } } when(active_out) { @@ -306,8 +307,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { io.tl.acquire.ready := Bool(true) when(io.tl.acquire.valid) { tl_buf_out(tl_cnt_out) := Mux(data_from_rel, - io.tl.release.bits.payload.data, - io.tl.acquire.bits.payload.data) + io.tl.release.bits.data, + io.tl.acquire.bits.data) } } when(!mif_done_out) { @@ -338,8 +339,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { io.mem.req_cmd.bits.tag := tag_out io.mem.req_cmd.bits.addr := addr_out io.mem.req_data.bits.data := Mux(data_from_rel, - io.tl.release.bits.payload.data, - io.tl.acquire.bits.payload.data) + io.tl.release.bits.data, + io.tl.acquire.bits.data) when(!active_out){ io.tl.release.ready := io.mem.req_data.ready io.tl.acquire.ready := io.mem.req_data.ready && !io.tl.release.valid @@ -352,12 +353,12 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { tl_done_out := tl_wrap_out when(io.tl.release.valid) { data_from_rel := Bool(true) - make_grant_ack := io.tl.release.bits.payload.requiresAck() - io.mem.req_data.bits.data := io.tl.release.bits.payload.data - val tag = Cat(io.tl.release.bits.header.src, - io.tl.release.bits.payload.client_xact_id, - io.tl.release.bits.payload.isVoluntary()) - val addr = io.tl.release.bits.payload.addr_block + make_grant_ack := io.tl.release.bits.requiresAck() + io.mem.req_data.bits.data := io.tl.release.bits.data + val tag = Cat(io.tl.release.bits.client_id, + io.tl.release.bits.client_xact_id, + io.tl.release.bits.isVoluntary()) + val addr = io.tl.release.bits.addr_block io.mem.req_cmd.bits.tag := tag io.mem.req_cmd.bits.addr := addr io.mem.req_cmd.bits.rw := rel_has_data @@ -367,12 +368,12 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { } .elsewhen(io.tl.acquire.valid) { data_from_rel := Bool(false) make_grant_ack := acq_has_data // i.e. is it a Put - io.mem.req_data.bits.data := io.tl.acquire.bits.payload.data + io.mem.req_data.bits.data := io.tl.acquire.bits.data io.mem.req_cmd.bits.rw := acq_has_data - val tag = Cat(io.tl.acquire.bits.header.src, - io.tl.acquire.bits.payload.client_xact_id, - io.tl.acquire.bits.payload.isBuiltInType()) - val addr = io.tl.acquire.bits.payload.addr_block + val tag = Cat(io.tl.acquire.bits.client_id, + io.tl.acquire.bits.client_xact_id, + io.tl.acquire.bits.isBuiltInType()) + val addr = io.tl.acquire.bits.addr_block io.mem.req_cmd.bits.tag := tag io.mem.req_cmd.bits.addr := addr io.mem.req_cmd.bits.rw := acq_has_data @@ -407,28 +408,25 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { // Aggregate incoming MemIO responses into TL Grants val active_in = Reg(init=Bool(false)) - val (tl_cnt_in, tl_wrap_in) = Counter(io.tl.grant.fire() && io.tl.grant.bits.payload.hasMultibeatData(), tlDataBeats) + val (tl_cnt_in, tl_wrap_in) = Counter(io.tl.grant.fire() && io.tl.grant.bits.hasMultibeatData(), tlDataBeats) val tag_in = Reg(UInt(width = mifTagBits)) if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) { val (mif_cnt_in, mif_wrap_in) = Counter(io.mem.resp.fire(), mifDataBeats) // TODO: Assumes all resps have data val mif_done_in = Reg(init=Bool(false)) val mif_buf_in = Vec.fill(mifDataBeats){ Reg(new MemData) } - val tl_buf_in = Vec.fill(tlDataBeats){ io.tl.acquire.bits.payload.data.clone } + val tl_buf_in = Vec.fill(tlDataBeats){ io.tl.acquire.bits.data.clone } tl_buf_in := tl_buf_in.fromBits(mif_buf_in.toBits) val tl_prog_in = (tl_cnt_in+UInt(1, width = log2Up(tlDataBeats+1)))*UInt(tlDataBits) val mif_prog_in = mif_cnt_in*UInt(mifDataBits) - gnt_arb.io.in(0).bits.payload := Grant( - is_builtin_type = tag_in(0), - g_type = Mux(tag_in(0), - Grant.getDataBlockType, - UInt(0)), // TODO: Assumes MI or MEI protocol - client_xact_id = tag_in >> UInt(1), - manager_xact_id = UInt(0), - addr_beat = tl_cnt_in, - data = tl_buf_in(tl_cnt_in)) - gnt_arb.io.in(0).bits.header.dst := (if(dstIdBits > 0) tag_in(dst_off, tlClientXactIdBits + 1) else UInt(0)) - gnt_arb.io.in(0).bits.header.src := UInt(0) + gnt_arb.io.in(0).bits := Grant( + dst = (if(dstIdBits > 0) tag_in(dst_off, tlClientXactIdBits + 1) else UInt(0)), + is_builtin_type = tag_in(0), + g_type = Mux(tag_in(0), Grant.getDataBlockType, UInt(0)), // TODO: Assumes MI or MEI protocol + client_xact_id = tag_in >> UInt(1), + manager_xact_id = UInt(0), + addr_beat = tl_cnt_in, + data = tl_buf_in(tl_cnt_in)) when(!active_in) { io.mem.resp.ready := Bool(true) @@ -453,17 +451,14 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { } else { // Don't generate all the uneeded data buffers and flow resp gnt_arb.io.in(0).valid := io.mem.resp.valid io.mem.resp.ready := gnt_arb.io.in(0).ready - gnt_arb.io.in(0).bits.payload := Grant( - is_builtin_type = io.mem.resp.bits.tag(0), - g_type = Mux(io.mem.resp.bits.tag(0), - Grant.getDataBlockType, - UInt(0)), // TODO: Assumes MI or MEI protocol - client_xact_id = io.mem.resp.bits.tag >> UInt(1), - manager_xact_id = UInt(0), - addr_beat = tl_cnt_in, - data = io.mem.resp.bits.data) - gnt_arb.io.in(0).bits.header.dst := (if(dstIdBits > 0) io.mem.resp.bits.tag(dst_off, tlClientXactIdBits + 1) else UInt(0)) - gnt_arb.io.in(0).bits.header.src := UInt(0) + gnt_arb.io.in(0).bits := Grant( + dst = (if(dstIdBits > 0) io.mem.resp.bits.tag(dst_off, tlClientXactIdBits + 1) else UInt(0)), + is_builtin_type = io.mem.resp.bits.tag(0), + g_type = Mux(io.mem.resp.bits.tag(0), Grant.getDataBlockType, UInt(0)), // TODO: Assumes MI or MEI protocol + client_xact_id = io.mem.resp.bits.tag >> UInt(1), + manager_xact_id = UInt(0), + addr_beat = tl_cnt_in, + data = io.mem.resp.bits.data) } } @@ -522,28 +517,28 @@ object HellaQueue } } -class MemPipeIOMemIOConverter(numRequests: Int, refillCycles: Int) extends Module { +class MemPipeIOMemIOConverter(numRequests: Int) extends MIFModule { val io = new Bundle { val cpu = new MemIO().flip val mem = new MemPipeIO } - val numEntries = numRequests * refillCycles + val numEntries = numRequests * mifDataBeats val size = log2Down(numEntries) + 1 val inc = Bool() val dec = Bool() val count = Reg(init=UInt(numEntries, size)) - val watermark = count >= UInt(refillCycles) + val watermark = count >= UInt(mifDataBeats) when (inc && !dec) { count := count + UInt(1) } when (!inc && dec) { - count := count - UInt(refillCycles) + count := count - UInt(mifDataBeats) } when (inc && dec) { - count := count - UInt(refillCycles-1) + count := count - UInt(mifDataBeats-1) } val cmdq_mask = io.cpu.req_cmd.bits.rw || watermark @@ -573,17 +568,17 @@ class MemPipeIOMemIOConverter(numRequests: Int, refillCycles: Int) extends Modul dec := io.mem.req_cmd.fire() && !io.mem.req_cmd.bits.rw } -class MemPipeIOTileLinkIOConverter(outstanding: Int, refillCycles: Int) extends Module { +class MemPipeIOTileLinkIOConverter(outstanding: Int) extends MIFModule { val io = new Bundle { - val tl = new TileLinkIO().flip + val tl = new ManagerTileLinkIO val mem = new MemPipeIO } val a = Module(new MemIOTileLinkIOConverter(1)) - val b = Module(new MemPipeIOMemIOConverter(outstanding, refillCycles)) + val b = Module(new MemPipeIOMemIOConverter(outstanding)) a.io.tl <> io.tl b.io.cpu.req_cmd <> Queue(a.io.mem.req_cmd, 2, pipe=true) - b.io.cpu.req_data <> Queue(a.io.mem.req_data, refillCycles, pipe=true) + b.io.cpu.req_data <> Queue(a.io.mem.req_data, mifDataBeats, pipe=true) a.io.mem.resp <> b.io.cpu.resp b.io.mem <> io.mem } diff --git a/uncore/src/main/scala/metadata.scala b/uncore/src/main/scala/metadata.scala index af5504c8..4e8082ee 100644 --- a/uncore/src/main/scala/metadata.scala +++ b/uncore/src/main/scala/metadata.scala @@ -108,17 +108,18 @@ class ManagerMetadata extends CoherenceMetadata { def requiresProbesOnVoluntaryWriteback(dummy: Int = 0): Bool = co.requiresProbes(M_FLUSH, this) - def makeProbe(cmd: UInt, addr_block: UInt): Probe = - Bundle(Probe(co.getProbeType(cmd, this), addr_block), { case TLId => id }) + def makeProbe(dst: UInt, cmd: UInt, addr_block: UInt): ProbeToDst = + Bundle(Probe(dst, co.getProbeType(cmd, this), addr_block), { case TLId => id }) - def makeProbe(acq: Acquire): Probe = - Bundle(Probe(co.getProbeType(acq, this), acq.addr_block), { case TLId => id }) + def makeProbe(dst: UInt, acq: Acquire): ProbeToDst = + Bundle(Probe(dst, co.getProbeType(acq, this), acq.addr_block), { case TLId => id }) - def makeProbeForVoluntaryWriteback(addr_block: UInt): Probe = - makeProbe(M_FLUSH, addr_block) + def makeProbeForVoluntaryWriteback(dst: UInt, addr_block: UInt): ProbeToDst = + makeProbe(dst, M_FLUSH, addr_block) - def makeGrant(rel: Release, manager_xact_id: UInt): Grant = { + def makeGrant(rel: ReleaseFromSrc, manager_xact_id: UInt): GrantToDst = { Bundle(Grant( + dst = rel.client_id, is_builtin_type = Bool(true), g_type = Grant.voluntaryAckType, client_xact_id = rel.client_xact_id, @@ -126,11 +127,12 @@ class ManagerMetadata extends CoherenceMetadata { } def makeGrant( - acq: Acquire, + acq: AcquireFromSrc, manager_xact_id: UInt, addr_beat: UInt = UInt(0), - data: UInt = UInt(0)): Grant = { + data: UInt = UInt(0)): GrantToDst = { Bundle(Grant( + dst = acq.client_id, is_builtin_type = acq.isBuiltInType(), g_type = Mux(acq.isBuiltInType(), acq.getBuiltInGrantType(), @@ -141,11 +143,24 @@ class ManagerMetadata extends CoherenceMetadata { data = data), { case TLId => id }) } - def onRelease(incoming: Release, src: UInt): ManagerMetadata = - Bundle(co.managerMetadataOnRelease(incoming, src, this), { case TLId => id }) + def makeGrant( + dst: UInt, + acq: AcquireFromSrc, + client_xact_id: UInt, + manager_xact_id: UInt, + addr_beat: UInt, + data: UInt): GrantToDst = { + val g = makeGrant(acq, manager_xact_id, addr_beat, data) + g.client_id := dst + g.client_xact_id := client_xact_id + g + } + + def onRelease(incoming: ReleaseFromSrc): ManagerMetadata = + Bundle(co.managerMetadataOnRelease(incoming, incoming.client_id, this), { case TLId => id }) - def onGrant(outgoing: Grant, dst: UInt): ManagerMetadata = - Bundle(co.managerMetadataOnGrant(outgoing, dst, this), { case TLId => id }) + def onGrant(outgoing: GrantToDst): ManagerMetadata = + Bundle(co.managerMetadataOnGrant(outgoing, outgoing.client_id, this), { case TLId => id }) } object ManagerMetadata { diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 1ed95df6..45cdb454 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -23,12 +23,14 @@ case object TLDataBits extends Field[Int] case object TLDataBeats extends Field[Int] case object TLNetworkIsOrderedP2P extends Field[Boolean] -abstract trait TileLinkParameters extends UsesParameters { +trait TileLinkParameters extends UsesParameters { val tlCoh = params(TLCoherencePolicy) val tlNManagers = params(TLNManagers) val tlNClients = params(TLNClients) val tlNCoherentClients = params(TLNCoherentClients) val tlNIncoherentClients = params(TLNIncoherentClients) + val tlClientIdBits = log2Up(tlNClients) + val tlManagerIdBits = log2Up(tlNManagers) val tlMaxClientXacts = params(TLMaxClientXacts) val tlMaxClientPorts = params(TLMaxClientPorts) val tlMaxManagerXacts = params(TLMaxManagerXacts) @@ -98,6 +100,10 @@ trait HasTileLinkData extends HasTileLinkBeatId { def hasMultibeatData(dummy: Int = 0): Bool } +trait HasClientId extends TLBundle { + val client_id = UInt(width = tlClientIdBits) +} + // Actual TileLink channel bundle definitions class Acquire extends ClientToManagerChannel @@ -350,12 +356,19 @@ class Probe extends ManagerToClientChannel } object Probe { - def apply(p_type: UInt, addr_block: UInt) = { + def apply(p_type: UInt, addr_block: UInt): Probe = { val prb = new Probe prb.p_type := p_type prb.addr_block := addr_block prb } + def apply(dst: UInt, p_type: UInt, addr_block: UInt): ProbeToDst = { + val prb = new ProbeToDst + prb.client_id := dst + prb.p_type := p_type + prb.addr_block := addr_block + prb + } } class Release extends ClientToManagerChannel @@ -435,8 +448,8 @@ object Grant { g_type: UInt, client_xact_id: UInt, manager_xact_id: UInt, - addr_beat: UInt = UInt(0), - data: UInt = UInt(0)): Grant = { + addr_beat: UInt, + data: UInt): Grant = { val gnt = new Grant gnt.is_builtin_type := is_builtin_type gnt.g_type := g_type @@ -446,6 +459,25 @@ object Grant { gnt.data := data gnt } + + def apply( + dst: UInt, + is_builtin_type: Bool, + g_type: UInt, + client_xact_id: UInt, + manager_xact_id: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): GrantToDst = { + val gnt = new GrantToDst + gnt.client_id := dst + gnt.is_builtin_type := is_builtin_type + gnt.g_type := g_type + gnt.client_xact_id := client_xact_id + gnt.manager_xact_id := manager_xact_id + gnt.addr_beat := addr_beat + gnt.data := data + gnt + } } class Finish extends ClientToManagerChannel with HasManagerTransactionId { @@ -453,7 +485,14 @@ class Finish extends ClientToManagerChannel with HasManagerTransactionId { def hasMultibeatData(dummy: Int = 0) = Bool(false) } -// Complete IO definitions for two types of TileLink clients +// These subtypes include a field for the source or destination ClientId +class AcquireFromSrc extends Acquire with HasClientId +class ProbeToDst extends Probe with HasClientId +class ReleaseFromSrc extends Release with HasClientId +class GrantToDst extends Grant with HasClientId + +// Complete IO definitions for two types of TileLink clients, including +// networking headers class UncachedTileLinkIO extends TLBundle { val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip @@ -479,28 +518,36 @@ class TileLinkIOWrapper extends TLModule { io.out.release.valid := Bool(false) } -// This version of TileLinkIO does not contain network headers for packets -// that originate in the Clients (i.e. Acquire and Release). These headers -// are provided in the top-level that instantiates the clients and network. +// This version of TileLinkIO does not contain network headers. The headers +// are provided in the top-level that instantiates the clients and network, +// probably using a TileLinkClientPort module. // By eliding the header subbundles within the clients we can enable // hierarchical P&R while minimizing unconnected port errors in GDS. // Secondly, this version of the interface elides Finish messages, with the // assumption that a FinishUnit has been coupled to the TileLinkIO port // to deal with acking received Grants. -class HeaderlessUncachedTileLinkIO extends TLBundle { +class ClientUncachedTileLinkIO extends TLBundle { val acquire = new DecoupledIO(new Acquire) val grant = new DecoupledIO(new Grant).flip } -class HeaderlessTileLinkIO extends HeaderlessUncachedTileLinkIO { +class ClientTileLinkIO extends ClientUncachedTileLinkIO { val probe = new DecoupledIO(new Probe).flip val release = new DecoupledIO(new Release) } -class HeaderlessTileLinkIOWrapper extends TLModule { +class ManagerTileLinkIO extends TLBundle { + val acquire = new DecoupledIO(new AcquireFromSrc).flip + val grant = new DecoupledIO(new GrantToDst) + val finish = new DecoupledIO(new Finish).flip + val probe = new DecoupledIO(new ProbeToDst) + val release = new DecoupledIO(new ReleaseFromSrc).flip +} + +class ClientTileLinkIOWrapper extends TLModule { val io = new Bundle { - val in = new HeaderlessUncachedTileLinkIO().flip - val out = new HeaderlessTileLinkIO + val in = new ClientUncachedTileLinkIO().flip + val out = new ClientTileLinkIO } io.out.acquire <> io.in.acquire io.out.grant <> io.in.grant @@ -509,17 +556,17 @@ class HeaderlessTileLinkIOWrapper extends TLModule { } object TileLinkIOWrapper { - def apply(utl: HeaderlessUncachedTileLinkIO, p: Parameters): HeaderlessTileLinkIO = { - val conv = Module(new HeaderlessTileLinkIOWrapper)(p) + def apply(utl: ClientUncachedTileLinkIO, p: Parameters): ClientTileLinkIO = { + val conv = Module(new ClientTileLinkIOWrapper)(p) conv.io.in <> utl conv.io.out } - def apply(utl: HeaderlessUncachedTileLinkIO): HeaderlessTileLinkIO = { - val conv = Module(new HeaderlessTileLinkIOWrapper) + def apply(utl: ClientUncachedTileLinkIO): ClientTileLinkIO = { + val conv = Module(new ClientTileLinkIOWrapper) conv.io.in <> utl conv.io.out } - def apply(tl: HeaderlessTileLinkIO): HeaderlessTileLinkIO = tl + def apply(tl: ClientTileLinkIO): ClientTileLinkIO = tl def apply(utl: UncachedTileLinkIO, p: Parameters): TileLinkIO = { val conv = Module(new TileLinkIOWrapper)(p) conv.io.in <> utl @@ -533,89 +580,6 @@ object TileLinkIOWrapper { def apply(tl: TileLinkIO): TileLinkIO = tl } -trait HasDataBeatCounters { - type HasBeat = TileLinkChannel with HasTileLinkBeatId - type HasClientId = TileLinkChannel with HasClientTransactionId - type LNAcquire = LogicalNetworkIO[Acquire] - type LNRelease = LogicalNetworkIO[Release] - type LNGrant = LogicalNetworkIO[Grant] - - def connectDataBeatCounter[S <: TileLinkChannel : ClassTag](inc: Bool, data: S, beat: UInt) = { - val multi = data.hasMultibeatData() - val (multi_cnt, multi_done) = Counter(inc && multi, data.tlDataBeats) - val cnt = Mux(multi, multi_cnt, beat) - val done = Mux(multi, multi_done, inc) - (cnt, done) - } - - def connectOutgoingDataBeatCounter[T <: Data : TypeTag]( - in: DecoupledIO[T], - beat: UInt = UInt(0)): (UInt, Bool) = { - in.bits match { - case p: TileLinkChannel if typeTag[T].tpe <:< typeTag[TileLinkChannel].tpe => - connectDataBeatCounter(in.fire(), p, beat) - case ln: LNGrant if typeTag[T].tpe <:< typeTag[LNGrant].tpe => - connectDataBeatCounter(in.fire(), ln.payload, beat) - case _ => { require(false, "Don't know how to connect a beat counter to " + typeTag[T].tpe); (UInt(0), Bool(false))} - } - } - - def connectIncomingDataBeatCounters[T <: HasClientId : ClassTag]( - in: DecoupledIO[LogicalNetworkIO[T]], - entries: Int, - getId: LogicalNetworkIO[T] => UInt): Vec[Bool] = { - Vec((0 until entries).map { i => - connectDataBeatCounter(in.fire() && getId(in.bits) === UInt(i), in.bits.payload, UInt(0))._2 - }) - } - - def connectIncomingDataBeatCounter[T <: Data : TypeTag](in: DecoupledIO[T]): Bool = { - in.bits match { - case p: TileLinkChannel if typeTag[T].tpe <:< typeTag[TileLinkChannel].tpe => - connectDataBeatCounter(in.fire(), p, UInt(0))._2 - case ln: LNAcquire if typeTag[T].tpe =:= typeTag[LNAcquire].tpe => - connectDataBeatCounter(in.fire(), ln.payload, UInt(0))._2 - case ln: LNRelease if typeTag[T].tpe =:= typeTag[LNRelease].tpe => - connectDataBeatCounter(in.fire(), ln.payload, UInt(0))._2 - case ln: LNGrant if typeTag[T].tpe =:= typeTag[LNGrant].tpe => - connectDataBeatCounter(in.fire(), ln.payload, UInt(0))._2 - case _ => { require(false, "Don't know how to connect a beat counter to " + typeTag[T].tpe); Bool(false)} - } - } - - def connectHeaderlessTwoWayBeatCounter[ T <: TileLinkChannel : ClassTag, S <: TileLinkChannel : ClassTag]( - max: Int, - up: DecoupledIO[T], - down: DecoupledIO[S], - beat: UInt): (Bool, UInt, Bool, UInt, Bool) = { - val cnt = Reg(init = UInt(0, width = log2Up(max+1))) - val (up_idx, do_inc) = connectDataBeatCounter(up.fire(), up.bits, beat) - val (down_idx, do_dec) = connectDataBeatCounter(down.fire(), down.bits, beat) - //Module.assert(!(do_dec && cnt === UInt(0)), "Decrementing 2way beat counter before ever incrementing") - cnt := Mux(do_dec, - Mux(do_inc, cnt, cnt - UInt(1)), - Mux(do_inc, cnt + UInt(1), cnt)) - (cnt > UInt(0), up_idx, do_inc, down_idx, do_dec) - } - - def connectTwoWayBeatCounter[ T <: TileLinkChannel : ClassTag, S <: TileLinkChannel : ClassTag]( - max: Int, - up: DecoupledIO[LogicalNetworkIO[T]], - down: DecoupledIO[LogicalNetworkIO[S]], - inc: T => Bool = (t: T) => Bool(true), - dec: S => Bool = (s: S) => Bool(true)): (Bool, UInt, Bool, UInt, Bool) = { - val cnt = Reg(init = UInt(0, width = log2Up(max+1))) - val (up_idx, up_done) = connectDataBeatCounter(up.fire(), up.bits.payload, UInt(0)) - val (down_idx, down_done) = connectDataBeatCounter(down.fire(), down.bits.payload, UInt(0)) - val do_inc = up_done && inc(up.bits.payload) - val do_dec = down_done && dec(down.bits.payload) - cnt := Mux(do_dec, - Mux(do_inc, cnt, cnt - UInt(1)), - Mux(do_inc, cnt + UInt(1), cnt)) - (cnt > UInt(0), up_idx, up_done, down_idx, down_done) - } -} - class FinishQueueEntry extends TLBundle { val fin = new Finish val dst = UInt(width = log2Up(params(LNEndpoints))) @@ -623,8 +587,7 @@ class FinishQueueEntry extends TLBundle { class FinishQueue(entries: Int) extends Queue(new FinishQueueEntry, entries) -class FinishUnit(srcId: Int = 0, outstanding: Int = 2) extends TLModule - with HasDataBeatCounters { +class FinishUnit(srcId: Int = 0, outstanding: Int = 2) extends TLModule with HasDataBeatCounters { val io = new Bundle { val grant = Decoupled(new LogicalNetworkIO(new Grant)).flip val refill = Decoupled(new Grant) @@ -645,12 +608,12 @@ class FinishUnit(srcId: Int = 0, outstanding: Int = 2) extends TLModule // a multibeat Grant. But Grants from multiple managers or transactions may // get interleaved, so we could need a counter for each. val done = if(tlNetworkDoesNotInterleaveBeats) { - connectIncomingDataBeatCounter(io.grant) + connectIncomingDataBeatCounterWithHeader(io.grant) } else { val entries = 1 << tlClientXactIdBits def getId(g: LogicalNetworkIO[Grant]) = g.payload.client_xact_id assert(getId(io.grant.bits) <= UInt(entries), "Not enough grant beat counters, only " + entries + " entries.") - connectIncomingDataBeatCounters(io.grant, entries, getId).reduce(_||_) + connectIncomingDataBeatCountersWithHeader(io.grant, entries, getId).reduce(_||_) } val q = Module(new FinishQueue(outstanding)) q.io.enq.valid := io.grant.fire() && g.requiresAck() && (!g.hasMultibeatData() || done) @@ -670,30 +633,7 @@ class FinishUnit(srcId: Int = 0, outstanding: Int = 2) extends TLModule } } -object TileLinkHeaderOverwriter { - def apply[T <: ClientToManagerChannel]( - in: DecoupledIO[LogicalNetworkIO[T]], - clientId: Int, - passThrough: Boolean): DecoupledIO[LogicalNetworkIO[T]] = { - val out = in.clone.asDirectionless - out.bits.payload := in.bits.payload - out.bits.header.src := UInt(clientId) - out.bits.header.dst := (if(passThrough) in.bits.header.dst else UInt(0)) - out.valid := in.valid - in.ready := out.ready - out - } - - def apply[T <: ClientToManagerChannel with HasCacheBlockAddress]( - in: DecoupledIO[LogicalNetworkIO[T]], - clientId: Int, - nBanks: Int, - addrConvert: UInt => UInt): DecoupledIO[LogicalNetworkIO[T]] = { - val out: DecoupledIO[LogicalNetworkIO[T]] = apply(in, clientId, false) - out.bits.header.dst := addrConvert(in.bits.payload.addr_block) - out - } - +object ClientTileLinkHeaderCreator { def apply[T <: ClientToManagerChannel with HasCacheBlockAddress : ClassTag]( in: DecoupledIO[T], clientId: Int, @@ -708,9 +648,24 @@ object TileLinkHeaderOverwriter { } } -class TileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) extends TLModule { +object ManagerTileLinkHeaderCreator { + def apply[T <: ManagerToClientChannel with HasClientId : ClassTag]( + in: DecoupledIO[T], + managerId: Int, + idConvert: UInt => UInt): DecoupledIO[LogicalNetworkIO[T]] = { + val out = new DecoupledIO(new LogicalNetworkIO(in.bits.clone)).asDirectionless + out.bits.payload := in.bits + out.bits.header.src := UInt(managerId) + out.bits.header.dst := idConvert(in.bits.client_id) + out.valid := in.valid + in.ready := out.ready + out + } +} + +class ClientTileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) extends TLModule { val io = new Bundle { - val client = new HeaderlessTileLinkIO().flip + val client = new ClientTileLinkIO().flip val network = new TileLinkIO } @@ -718,8 +673,8 @@ class TileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) extends TLMo finisher.io.grant <> io.network.grant io.network.finish <> finisher.io.finish - val acq_with_header = TileLinkHeaderOverwriter(io.client.acquire, clientId, addrConvert) - val rel_with_header = TileLinkHeaderOverwriter(io.client.release, clientId, addrConvert) + val acq_with_header = ClientTileLinkHeaderCreator(io.client.acquire, clientId, addrConvert) + val rel_with_header = ClientTileLinkHeaderCreator(io.client.release, clientId, addrConvert) val prb_without_header = DecoupledLogicalNetworkIOUnwrapper(io.network.probe) val gnt_without_header = finisher.io.refill @@ -731,58 +686,63 @@ class TileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) extends TLMo io.client.grant <> gnt_without_header } -object TileLinkNetworkPort { - def apply[T <: Data]( - client: HeaderlessTileLinkIO, - clientId: Int = 0, - addrConvert: UInt => UInt = u => UInt(0))(implicit p: Parameters): TileLinkIO = { - val port = Module(new TileLinkNetworkPort(clientId, addrConvert))(p) - port.io.client <> client - port.io.network +class ManagerTileLinkNetworkPort(managerId: Int, idConvert: UInt => UInt) extends TLModule { + val io = new Bundle { + val manager = new ManagerTileLinkIO().flip + val network = new TileLinkIO().flip } + io.network.grant <> ManagerTileLinkHeaderCreator(io.manager.grant, managerId, (u: UInt) => u) + io.network.probe <> ManagerTileLinkHeaderCreator(io.manager.probe, managerId, idConvert) + io.manager.acquire.bits.client_id := io.network.acquire.bits.header.src + io.manager.acquire <> DecoupledLogicalNetworkIOUnwrapper(io.network.acquire) + io.manager.release.bits.client_id := io.network.release.bits.header.src + io.manager.release <> DecoupledLogicalNetworkIOUnwrapper(io.network.release) + io.manager.finish <> DecoupledLogicalNetworkIOUnwrapper(io.network.finish) } -class TileLinkEnqueuer(depths: (Int, Int, Int, Int, Int)) extends Module { +case class TileLinkDepths(acq: Int, prb: Int, rel: Int, gnt: Int, fin: Int) + +class TileLinkEnqueuer(depths: TileLinkDepths) extends Module { val io = new Bundle { val client = new TileLinkIO().flip val manager = new TileLinkIO } - io.manager.acquire <> (if(depths._1 > 0) Queue(io.client.acquire, depths._1) else io.client.acquire) - io.client.probe <> (if(depths._2 > 0) Queue(io.manager.probe, depths._2) else io.manager.probe) - io.manager.release <> (if(depths._3 > 0) Queue(io.client.release, depths._3) else io.client.release) - io.client.grant <> (if(depths._4 > 0) Queue(io.manager.grant, depths._4) else io.manager.grant) - io.manager.finish <> (if(depths._5 > 0) Queue(io.client.finish, depths._5) else io.client.finish) + io.manager.acquire <> (if(depths.acq > 0) Queue(io.client.acquire, depths.acq) else io.client.acquire) + io.client.probe <> (if(depths.prb > 0) Queue(io.manager.probe, depths.prb) else io.manager.probe) + io.manager.release <> (if(depths.rel > 0) Queue(io.client.release, depths.rel) else io.client.release) + io.client.grant <> (if(depths.gnt > 0) Queue(io.manager.grant, depths.gnt) else io.manager.grant) + io.manager.finish <> (if(depths.fin > 0) Queue(io.client.finish, depths.fin) else io.client.finish) } object TileLinkEnqueuer { - def apply(in: TileLinkIO, depths: (Int, Int, Int, Int, Int))(p: Parameters): TileLinkIO = { + def apply(in: TileLinkIO, depths: TileLinkDepths)(p: Parameters): TileLinkIO = { val t = Module(new TileLinkEnqueuer(depths))(p) t.io.client <> in t.io.manager } def apply(in: TileLinkIO, depth: Int)(p: Parameters): TileLinkIO = { - apply(in, (depth, depth, depth, depth, depth))(p) + apply(in, TileLinkDepths(depth, depth, depth, depth, depth))(p) } } -abstract trait HasArbiterTypes { +/** Utility functions for constructing TileLinkIO arbiters */ +trait TileLinkArbiterLike extends TileLinkParameters { + // Some shorthand type variables type ManagerSourcedWithId = ManagerToClientChannel with HasClientTransactionId type ClientSourcedWithId = ClientToManagerChannel with HasClientTransactionId - type ClientSourcedWithIdAndData = ClientToManagerChannel with - HasClientTransactionId with - HasTileLinkData -} + type ClientSourcedWithIdAndData = ClientToManagerChannel with HasClientTransactionId with HasTileLinkData -// Utility functions for constructing TileLinkIO arbiters -trait TileLinkArbiterLike extends HasArbiterTypes with TileLinkParameters{ - val arbN: Int - // These are filled in depending on whether the arbiter mucks with the - // client ids and then needs to revert them on the way back + val arbN: Int // The number of ports on the client side + + // These abstract funcs are filled in depending on whether the arbiter mucks with the + // outgoing client ids to track sourcing and then needs to revert them on the way back def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int): Bits def managerSourcedClientXactId(in: ManagerSourcedWithId): Bits def arbIdx(in: ManagerSourcedWithId): UInt - def hookupClientSource[M <: ClientSourcedWithIdAndData : ClassTag]( + // The following functions are all wiring helpers for each of the different types of TileLink channels + + def hookupClientSource[M <: ClientSourcedWithIdAndData]( clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], mngr: DecoupledIO[LogicalNetworkIO[M]]) { def hasData(m: LogicalNetworkIO[M]) = m.payload.hasMultibeatData() @@ -796,7 +756,7 @@ trait TileLinkArbiterLike extends HasArbiterTypes with TileLinkParameters{ arb.io.out <> mngr } - def hookupClientSourceHeaderless[M <: ClientSourcedWithIdAndData : ClassTag]( + def hookupClientSourceHeaderless[M <: ClientSourcedWithIdAndData]( clts: Seq[DecoupledIO[M]], mngr: DecoupledIO[M]) { def hasData(m: M) = m.hasMultibeatData() @@ -854,23 +814,20 @@ trait TileLinkArbiterLike extends HasArbiterTypes with TileLinkParameters{ } } - def hookupManagerSourceBroadcast[M <: Data]( - clts: Seq[DecoupledIO[M]], - mngr: DecoupledIO[M]) { + def hookupManagerSourceBroadcast[M <: Data](clts: Seq[DecoupledIO[M]], mngr: DecoupledIO[M]) { clts.map{ _.valid := mngr.valid } clts.map{ _.bits := mngr.bits } mngr.ready := clts.map(_.ready).reduce(_&&_) } - def hookupFinish[M <: LogicalNetworkIO[Finish] : ClassTag]( - clts: Seq[DecoupledIO[M]], - mngr: DecoupledIO[M]) { + def hookupFinish[M <: LogicalNetworkIO[Finish]]( clts: Seq[DecoupledIO[M]], mngr: DecoupledIO[M]) { val arb = Module(new RRArbiter(mngr.bits.clone, arbN)) - arb.io.in zip clts map { case (arb, req) => arb <> req } + arb.io.in <> clts arb.io.out <> mngr } } +/** Abstract base case for any Arbiters that have UncachedTileLinkIOs */ abstract class UncachedTileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiterLike { val io = new Bundle { val in = Vec.fill(arbN){new UncachedTileLinkIO}.flip @@ -881,6 +838,7 @@ abstract class UncachedTileLinkIOArbiter(val arbN: Int) extends Module with Tile hookupManagerSourceWithId(io.in.map(_.grant), io.out.grant) } +/** Abstract base case for any Arbiters that have cached TileLinkIOs */ abstract class TileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiterLike { val io = new Bundle { val in = Vec.fill(arbN){new TileLinkIO}.flip @@ -893,31 +851,7 @@ abstract class TileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbi hookupManagerSourceWithId(io.in.map(_.grant), io.out.grant) } -class HeaderlessUncachedTileLinkIOArbiter(val arbN: Int) extends Module - with TileLinkArbiterLike - with AppendsArbiterId { - val io = new Bundle { - val in = Vec.fill(arbN){new HeaderlessUncachedTileLinkIO}.flip - val out = new HeaderlessUncachedTileLinkIO - } - hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) - hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) -} - -class HeaderlessTileLinkIOArbiter(val arbN: Int) extends Module - with TileLinkArbiterLike - with AppendsArbiterId { - val io = new Bundle { - val in = Vec.fill(arbN){new HeaderlessTileLinkIO}.flip - val out = new HeaderlessTileLinkIO - } - hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) - hookupClientSourceHeaderless(io.in.map(_.release), io.out.release) - hookupManagerSourceBroadcast(io.in.map(_.probe), io.out.probe) - hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) -} - -// Appends the port index of the arbiter to the client_xact_id +/** Appends the port index of the arbiter to the client_xact_id */ trait AppendsArbiterId extends TileLinkArbiterLike { def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = Cat(in.client_xact_id, UInt(id, log2Up(arbN))) @@ -926,24 +860,125 @@ trait AppendsArbiterId extends TileLinkArbiterLike { def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id(log2Up(arbN)-1,0).toUInt } -// Uses the client_xact_id as is (assumes it has been set to port index) +/** Uses the client_xact_id as is (assumes it has been set to port index) */ trait PassesId extends TileLinkArbiterLike { def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = in.client_xact_id def managerSourcedClientXactId(in: ManagerSourcedWithId) = in.client_xact_id def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id } -// Overwrites some default client_xact_id with the port idx +/** Overwrites some default client_xact_id with the port idx */ trait UsesNewId extends TileLinkArbiterLike { def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = UInt(id, log2Up(arbN)) def managerSourcedClientXactId(in: ManagerSourcedWithId) = UInt(0) def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id } -// Mix-in id generation traits to make concrete arbiter classes +// Now we can mix-in thevarious id-generation traits to make concrete arbiter classes class UncachedTileLinkIOArbiterThatAppendsArbiterId(val n: Int) extends UncachedTileLinkIOArbiter(n) with AppendsArbiterId class UncachedTileLinkIOArbiterThatPassesId(val n: Int) extends UncachedTileLinkIOArbiter(n) with PassesId class UncachedTileLinkIOArbiterThatUsesNewId(val n: Int) extends UncachedTileLinkIOArbiter(n) with UsesNewId class TileLinkIOArbiterThatAppendsArbiterId(val n: Int) extends TileLinkIOArbiter(n) with AppendsArbiterId class TileLinkIOArbiterThatPassesId(val n: Int) extends TileLinkIOArbiter(n) with PassesId class TileLinkIOArbiterThatUsesNewId(val n: Int) extends TileLinkIOArbiter(n) with UsesNewId + +/** Concrete uncached client-side arbiter that appends the arbiter's port id to client_xact_id */ +class ClientUncachedTileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiterLike with AppendsArbiterId { + val io = new Bundle { + val in = Vec.fill(arbN){new ClientUncachedTileLinkIO}.flip + val out = new ClientUncachedTileLinkIO + } + hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) + hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) +} + +/** Concrete client-side arbiter that appends the arbiter's port id to client_xact_id */ +class ClientTileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiterLike with AppendsArbiterId { + val io = new Bundle { + val in = Vec.fill(arbN){new ClientTileLinkIO}.flip + val out = new ClientTileLinkIO + } + hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) + hookupClientSourceHeaderless(io.in.map(_.release), io.out.release) + hookupManagerSourceBroadcast(io.in.map(_.probe), io.out.probe) + hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) +} + +/** Utility trait containing wiring functions to keep track of how many data beats have + * been sent or recieved over a particular TileLinkChannel or pair of channels. + * + * Won't count message types that don't have data. + * Used in XactTrackers and FinishUnit. + */ +trait HasDataBeatCounters { + type HasBeat = TileLinkChannel with HasTileLinkBeatId + + /** Returns the current count on this channel and when a message is done + * @param inc increment the counter (usually .valid or .fire()) + * @param data the actual channel data + * @param beat count to return for single-beat messages + */ + def connectDataBeatCounter[S <: TileLinkChannel](inc: Bool, data: S, beat: UInt) = { + val multi = data.hasMultibeatData() + val (multi_cnt, multi_done) = Counter(inc && multi, data.tlDataBeats) + val cnt = Mux(multi, multi_cnt, beat) + val done = Mux(multi, multi_done, inc) + (cnt, done) + } + + /** Counter for beats on outgoing DecoupledIOs */ + def connectOutgoingDataBeatCounter[T <: TileLinkChannel](in: DecoupledIO[T], beat: UInt = UInt(0)): (UInt, Bool) = + connectDataBeatCounter(in.fire(), in.bits, beat) + + /** Returns done but not cnt. Use the addr_beat subbundle instead of cnt for beats on + * incoming channels in case of network reordering. + */ + def connectIncomingDataBeatCounter[T <: TileLinkChannel](in: DecoupledIO[T]): Bool = + connectDataBeatCounter(in.fire(), in.bits, UInt(0))._2 + + /** Counter for beats on incoming DecoupledIO[LogicalNetworkIO[]]s returns done */ + def connectIncomingDataBeatCounterWithHeader[T <: TileLinkChannel](in: DecoupledIO[LogicalNetworkIO[T]]): Bool = + connectDataBeatCounter(in.fire(), in.bits.payload, UInt(0))._2 + + /** If the network might interleave beats from different messages, we need a Vec of counters, + * one for every outstanding message id that might be interleaved. + * + * @param getId mapping from Message to counter id + */ + def connectIncomingDataBeatCountersWithHeader[T <: TileLinkChannel with HasClientTransactionId]( + in: DecoupledIO[LogicalNetworkIO[T]], + entries: Int, + getId: LogicalNetworkIO[T] => UInt): Vec[Bool] = { + Vec((0 until entries).map { i => + connectDataBeatCounter(in.fire() && getId(in.bits) === UInt(i), in.bits.payload, UInt(0))._2 + }) + } + + /** Provides counters on two channels, as well a meta-counter that tracks how many + * messages have been sent over the up channel but not yet responded to over the down channel + * + * @param max max number of outstanding ups with no down + * @param up outgoing channel + * @param down incoming channel + * @param beat overrides cnts on single-beat messages + * @param track whether up's message should be tracked + * @return a tuple containing whether their are outstanding messages, up's count, + * up's done, down's count, down's done + */ + def connectTwoWayBeatCounter[T <: TileLinkChannel, S <: TileLinkChannel]( + max: Int, + up: DecoupledIO[T], + down: DecoupledIO[S], + beat: UInt = UInt(0), + track: T => Bool = (t: T) => Bool(true)): (Bool, UInt, Bool, UInt, Bool) = { + val cnt = Reg(init = UInt(0, width = log2Up(max+1))) + val (up_idx, up_done) = connectDataBeatCounter(up.fire(), up.bits, beat) + val (down_idx, down_done) = connectDataBeatCounter(down.fire(), down.bits, beat) + val do_inc = up_done && track(up.bits) + val do_dec = down_done + cnt := Mux(do_dec, + Mux(do_inc, cnt, cnt - UInt(1)), + Mux(do_inc, cnt + UInt(1), cnt)) + (cnt > UInt(0), up_idx, up_done, down_idx, down_done) + } +} diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 9d1c93b7..8dda15b7 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -32,19 +32,19 @@ abstract class CoherenceAgentBundle extends Bundle with CoherenceAgentParameters abstract class CoherenceAgentModule extends Module with CoherenceAgentParameters trait HasCoherenceAgentWiringHelpers { - def doOutputArbitration[T <: TileLinkChannel : ClassTag]( - out: DecoupledIO[LogicalNetworkIO[T]], - ins: Seq[DecoupledIO[LogicalNetworkIO[T]]]) { - def lock(o: LogicalNetworkIO[T]) = o.payload.hasMultibeatData() - val arb = Module(new LockingRRArbiter( out.bits.clone, ins.size, out.bits.payload.tlDataBeats, lock _)) + def doOutputArbitration[T <: TileLinkChannel]( + out: DecoupledIO[T], + ins: Seq[DecoupledIO[T]]) { + def lock(o: T) = o.hasMultibeatData() + val arb = Module(new LockingRRArbiter(out.bits.clone, ins.size, out.bits.tlDataBeats, lock _)) out <> arb.io.out - arb.io.in zip ins map { case (a, in) => a <> in } + arb.io.in <> ins } def doInputRouting[T <: HasManagerTransactionId]( - in: DecoupledIO[LogicalNetworkIO[T]], - outs: Seq[DecoupledIO[LogicalNetworkIO[T]]]) { - val idx = in.bits.payload.manager_xact_id + in: DecoupledIO[T], + outs: Seq[DecoupledIO[T]]) { + val idx = in.bits.manager_xact_id outs.map(_.bits := in.bits) outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && idx === UInt(i) } in.ready := Vec(outs.map(_.ready)).read(idx) @@ -52,23 +52,23 @@ trait HasCoherenceAgentWiringHelpers { } trait HasInnerTLIO extends CoherenceAgentBundle { - val inner = Bundle(new TileLinkIO)(innerTLParams).flip + val inner = Bundle(new ManagerTileLinkIO)(innerTLParams) val incoherent = Vec.fill(inner.tlNCoherentClients){Bool()}.asInput - def iacq(dummy: Int = 0) = inner.acquire.bits.payload - def iprb(dummy: Int = 0) = inner.probe.bits.payload - def irel(dummy: Int = 0) = inner.release.bits.payload - def ignt(dummy: Int = 0) = inner.grant.bits.payload - def ifin(dummy: Int = 0) = inner.finish.bits.payload + def iacq(dummy: Int = 0) = inner.acquire.bits + def iprb(dummy: Int = 0) = inner.probe.bits + def irel(dummy: Int = 0) = inner.release.bits + def ignt(dummy: Int = 0) = inner.grant.bits + def ifin(dummy: Int = 0) = inner.finish.bits } trait HasUncachedOuterTLIO extends CoherenceAgentBundle { - val outer = Bundle(new HeaderlessUncachedTileLinkIO)(outerTLParams) + val outer = Bundle(new ClientUncachedTileLinkIO)(outerTLParams) def oacq(dummy: Int = 0) = outer.acquire.bits def ognt(dummy: Int = 0) = outer.grant.bits } trait HasCachedOuterTLIO extends CoherenceAgentBundle { - val outer = Bundle(new HeaderlessTileLinkIO)(outerTLParams) + val outer = Bundle(new ClientTileLinkIO)(outerTLParams) def oacq(dummy: Int = 0) = outer.acquire.bits def oprb(dummy: Int = 0) = outer.probe.bits def orel(dummy: Int = 0) = outer.release.bits @@ -78,8 +78,8 @@ trait HasCachedOuterTLIO extends CoherenceAgentBundle { class ManagerTLIO extends HasInnerTLIO with HasUncachedOuterTLIO abstract class CoherenceAgent extends CoherenceAgentModule { - def innerTL: TileLinkIO - def outerTL: HeaderlessTileLinkIO + def innerTL: ManagerTileLinkIO + def outerTL: ClientTileLinkIO def incoherent: Vec[Bool] } @@ -109,49 +109,25 @@ trait HasTrackerConflictIO extends Bundle { class ManagerXactTrackerIO extends ManagerTLIO with HasTrackerConflictIO class HierarchicalXactTrackerIO extends HierarchicalTLIO with HasTrackerConflictIO -abstract class XactTracker extends CoherenceAgentModule - with HasDataBeatCounters { +abstract class XactTracker extends CoherenceAgentModule with HasDataBeatCounters { def addPendingBitWhenBeat[T <: HasBeat](inc: Bool, in: T): UInt = Fill(in.tlDataBeats, inc) & UIntToOH(in.addr_beat) def dropPendingBitWhenBeat[T <: HasBeat](dec: Bool, in: T): UInt = ~Fill(in.tlDataBeats, dec) | ~UIntToOH(in.addr_beat) - def addPendingBitWhenBeatHasData[T <: Data : TypeTag](in: DecoupledIO[T]): UInt = { - in.bits match { - case p: HasBeat if typeTag[T].tpe <:< typeTag[HasBeat].tpe => - addPendingBitWhenBeat(in.fire() && p.hasData(), p) - case ln: LNAcquire if typeTag[T].tpe <:< typeTag[LNAcquire].tpe => - addPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) - case ln: LNRelease if typeTag[T].tpe <:< typeTag[LNRelease].tpe => - addPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) - case ln: LNGrant if typeTag[T].tpe <:< typeTag[LNGrant].tpe => - addPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) - case _ => { require(false, "Don't know how track beats of " + typeTag[T].tpe); UInt(0) } - } - } + def addPendingBitWhenBeatHasData[T <: HasBeat](in: DecoupledIO[T]): UInt = + addPendingBitWhenBeat(in.fire() && in.bits.hasData(), in.bits) - def addPendingBitWhenBeatIsGetOrAtomic(in: DecoupledIO[LogicalNetworkIO[Acquire]]): UInt = { - val a = in.bits.payload + def addPendingBitWhenBeatIsGetOrAtomic(in: DecoupledIO[AcquireFromSrc]): UInt = { + val a = in.bits val isGetOrAtomic = a.isBuiltInType() && - (Vec(Acquire.getType, Acquire.getBlockType, Acquire.putAtomicType).contains(a.a_type)) - addPendingBitWhenBeat(in.fire() && isGetOrAtomic, in.bits.payload) + (Vec(Acquire.getType, Acquire.getBlockType, Acquire.putAtomicType).contains(a.a_type)) + addPendingBitWhenBeat(in.fire() && isGetOrAtomic, a) } - def dropPendingBitWhenBeatHasData[T <: Data : TypeTag](in: DecoupledIO[T]): UInt = { - in.bits match { - case p: HasBeat if typeTag[T].tpe <:< typeTag[HasBeat].tpe => - dropPendingBitWhenBeat(in.fire() && p.hasData(), p) - case ln: LNAcquire if typeTag[T].tpe <:< typeTag[LNAcquire].tpe => - dropPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) - case ln: LNRelease if typeTag[T].tpe <:< typeTag[LNRelease].tpe => - dropPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) - case ln: LNGrant if typeTag[T].tpe <:< typeTag[LNGrant].tpe => - dropPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) - case _ => { require(false, "Don't know how track beats of " + typeTag[T].tpe); UInt(0) } - } - } + def dropPendingBitWhenBeatHasData[T <: HasBeat](in: DecoupledIO[T]): UInt = + dropPendingBitWhenBeat(in.fire() && in.bits.hasData(), in.bits) - def dropPendingBitAtDest(in: DecoupledIO[LogicalNetworkIO[Probe]]): UInt = { - ~Fill(in.bits.payload.tlNCoherentClients, in.fire()) | ~UIntToOH(in.bits.header.dst) - } + def dropPendingBitAtDest(in: DecoupledIO[ProbeToDst]): UInt = + ~Fill(in.bits.tlNCoherentClients, in.fire()) | ~UIntToOH(in.bits.client_id) } From 6d40a610602e3250cd247cacb5d84722914f01d5 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 19 Apr 2015 22:06:44 -0700 Subject: [PATCH 368/374] TileLink scala doc and parameter renaming --- uncore/src/main/scala/broadcast.scala | 8 +- uncore/src/main/scala/cache.scala | 8 +- uncore/src/main/scala/metadata.scala | 10 +- uncore/src/main/scala/tilelink.scala | 500 +++++++++++++++++++------- uncore/src/main/scala/uncore.scala | 10 +- 5 files changed, 381 insertions(+), 155 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 339a7540..b554b231 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -216,13 +216,13 @@ class BroadcastAcquireTracker(trackerId: Int) extends BroadcastXactTracker { Acquire.prefetchType).contains(xact.a_type)), "Broadcast Hub does not support PutAtomics, subblock Gets/Puts, or prefetches") // TODO - val release_count = Reg(init=UInt(0, width = log2Up(io.inner.tlNCoherentClients+1))) - val pending_probes = Reg(init=Bits(0, width = io.inner.tlNCoherentClients)) + val release_count = Reg(init=UInt(0, width = log2Up(io.inner.tlNCachingClients+1))) + val pending_probes = Reg(init=Bits(0, width = io.inner.tlNCachingClients)) val curr_p_id = PriorityEncoder(pending_probes) val full_sharers = coh.full() val probe_self = io.inner.acquire.bits.requiresSelfProbe() - val mask_self_true = UInt(UInt(1) << io.inner.acquire.bits.client_id, width = io.inner.tlNCoherentClients) - val mask_self_false = ~UInt(UInt(1) << io.inner.acquire.bits.client_id, width = io.inner.tlNCoherentClients) + val mask_self_true = UInt(UInt(1) << io.inner.acquire.bits.client_id, width = io.inner.tlNCachingClients) + val mask_self_false = ~UInt(UInt(1) << io.inner.acquire.bits.client_id, width = io.inner.tlNCachingClients) val mask_self = Mux(probe_self, full_sharers | mask_self_true, full_sharers & mask_self_false) val mask_incoherent = mask_self & ~io.incoherent.toBits diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 8ae4d00b..75bb9e1f 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -625,7 +625,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { // State holding progress made on processing this transaction val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) val pending_irels = connectTwoWayBeatCounter( - max = io.inner.tlNCoherentClients, + max = io.inner.tlNCachingClients, up = io.inner.probe, down = io.inner.release)._1 val (pending_ognts, oacq_data_idx, oacq_data_done, ognt_data_idx, ognt_data_done) = @@ -641,7 +641,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { down = io.inner.finish, track = (g: Grant) => g.requiresAck())._1 val pending_puts = Reg(init=Bits(0, width = io.inner.tlDataBeats)) - val pending_iprbs = Reg(init = Bits(0, width = io.inner.tlNCoherentClients)) + val pending_iprbs = Reg(init = Bits(0, width = io.inner.tlNCachingClients)) val pending_reads = Reg(init=Bits(0, width = io.inner.tlDataBeats)) val pending_writes = Reg(init=Bits(0, width = io.inner.tlDataBeats)) val pending_resps = Reg(init=Bits(0, width = io.inner.tlDataBeats)) @@ -1009,8 +1009,8 @@ class L2WritebackUnit(trackerId: Int) extends L2XactTracker { val xact_id = Reg{ UInt() } val irel_had_data = Reg(init = Bool(false)) - val irel_cnt = Reg(init = UInt(0, width = log2Up(io.inner.tlNCoherentClients+1))) - val pending_probes = Reg(init = Bits(0, width = io.inner.tlNCoherentClients)) + val irel_cnt = Reg(init = UInt(0, width = log2Up(io.inner.tlNCachingClients+1))) + val pending_probes = Reg(init = Bits(0, width = io.inner.tlNCachingClients)) val curr_probe_dst = PriorityEncoder(pending_probes) val full_sharers = io.wb.req.bits.coh.inner.full() val mask_incoherent = full_sharers & ~io.incoherent.toBits diff --git a/uncore/src/main/scala/metadata.scala b/uncore/src/main/scala/metadata.scala index 4e8082ee..cf52d286 100644 --- a/uncore/src/main/scala/metadata.scala +++ b/uncore/src/main/scala/metadata.scala @@ -9,11 +9,11 @@ abstract class CoherenceMetadata extends Bundle { val id = params(TLId) } -/* The ClientMetadata stores the client-side coherence information, - such as permissions on the data and whether the data is dirty. - Its API can be used to make TileLink messages in response to - memory operations or TileLink Probes. -*/ +/** Stores the client-side coherence information, + * such as permissions on the data and whether the data is dirty. + * Its API can be used to make TileLink messages in response to + * memory operations or [[uncore.Probe]] messages. + */ class ClientMetadata extends CoherenceMetadata { val state = UInt(width = co.clientStateWidth) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 45cdb454..b759d48b 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -3,38 +3,50 @@ package uncore import Chisel._ import scala.math.max -import scala.reflect._ -import scala.reflect.runtime.universe._ -// Parameters exposed to the top-level design, set based on -// external requirements or design space exploration -// -case object TLId extends Field[String] // Unique name per network +/** Parameters exposed to the top-level design, set based on + * external requirements or design space exploration + */ +/** Unique name per TileLink network*/ +case object TLId extends Field[String] +/** Coherency policy used to define custom mesage types */ case object TLCoherencePolicy extends Field[CoherencePolicy] -case object TLNManagers extends Field[Int] +/** Number of manager agents */ +case object TLNManagers extends Field[Int] +/** Number of client agents */ case object TLNClients extends Field[Int] -case object TLNCoherentClients extends Field[Int] -case object TLNIncoherentClients extends Field[Int] +/** Number of client agents that cache data and use custom [[uncore.Acquire]] types */ +case object TLNCachingClients extends Field[Int] +/** Number of client agents that do not cache data and use built-in [[uncoreAcquire]] types */ +case object TLNCachelessClients extends Field[Int] +/** Maximum number of unique outstanding transactions per client */ case object TLMaxClientXacts extends Field[Int] -case object TLMaxClientPorts extends Field[Int] +/** Maximum number of clients multiplexed onto a single port */ +case object TLMaxClientsPerPort extends Field[Int] +/** Maximum number of unique outstanding transactions per manager */ case object TLMaxManagerXacts extends Field[Int] +/** Width of cache block addresses */ case object TLBlockAddrBits extends Field[Int] +/** Width of data beats */ case object TLDataBits extends Field[Int] +/** Number of data beats per cache block */ case object TLDataBeats extends Field[Int] +/** Whether the underlying physical network preserved point-to-point ordering of messages */ case object TLNetworkIsOrderedP2P extends Field[Boolean] +/** Utility trait for building Modules and Bundles that use TileLink parameters */ trait TileLinkParameters extends UsesParameters { val tlCoh = params(TLCoherencePolicy) val tlNManagers = params(TLNManagers) val tlNClients = params(TLNClients) - val tlNCoherentClients = params(TLNCoherentClients) - val tlNIncoherentClients = params(TLNIncoherentClients) + val tlNCachingClients = params(TLNCachingClients) + val tlNCachelessClients = params(TLNCachelessClients) val tlClientIdBits = log2Up(tlNClients) val tlManagerIdBits = log2Up(tlNManagers) val tlMaxClientXacts = params(TLMaxClientXacts) - val tlMaxClientPorts = params(TLMaxClientPorts) + val tlMaxClientsPerPort = params(TLMaxClientsPerPort) val tlMaxManagerXacts = params(TLMaxManagerXacts) - val tlClientXactIdBits = log2Up(tlMaxClientXacts*tlMaxClientPorts) + val tlClientXactIdBits = log2Up(tlMaxClientXacts*tlMaxClientsPerPort) val tlManagerXactIdBits = log2Up(tlMaxManagerXacts) val tlBlockAddrBits = params(TLBlockAddrBits) val tlDataBits = params(TLDataBits) @@ -61,19 +73,23 @@ trait TileLinkParameters extends UsesParameters { abstract class TLBundle extends Bundle with TileLinkParameters abstract class TLModule extends Module with TileLinkParameters -// Directionality of message channel -// Used to hook up logical network ports to physical network ports +/** Base trait for all TileLink channels */ trait TileLinkChannel extends TLBundle { def hasData(dummy: Int = 0): Bool def hasMultibeatData(dummy: Int = 0): Bool } +/** Directionality of message channel. Used to hook up logical network ports to physical network ports */ trait ClientToManagerChannel extends TileLinkChannel +/** Directionality of message channel. Used to hook up logical network ports to physical network ports */ trait ManagerToClientChannel extends TileLinkChannel +/** Directionality of message channel. Used to hook up logical network ports to physical network ports */ trait ClientToClientChannel extends TileLinkChannel // Unused for now -// Common signals that are used in multiple channels. -// These traits are useful for type parameterizing bundle wiring functions. -// +/** Common signals that are used in multiple channels. + * These traits are useful for type parameterizing bundle wiring functions. + */ + +/** Address of a cache block. */ trait HasCacheBlockAddress extends TLBundle { val addr_block = UInt(width = tlBlockAddrBits) @@ -81,18 +97,22 @@ trait HasCacheBlockAddress extends TLBundle { def conflicts(addr: UInt) = this.addr_block === addr } +/** Sub-block address or beat id of multi-beat data */ trait HasTileLinkBeatId extends TLBundle { val addr_beat = UInt(width = tlBeatAddrBits) } +/* Client-side transaction id. Usually Miss Status Handling Register File index */ trait HasClientTransactionId extends TLBundle { val client_xact_id = Bits(width = tlClientXactIdBits) } +/** Manager-side transaction id. Usually Transaction Status Handling Register File index. */ trait HasManagerTransactionId extends TLBundle { val manager_xact_id = Bits(width = tlManagerXactIdBits) } +/** A single beat of cache block data */ trait HasTileLinkData extends HasTileLinkBeatId { val data = UInt(width = tlDataBits) @@ -100,61 +120,90 @@ trait HasTileLinkData extends HasTileLinkBeatId { def hasMultibeatData(dummy: Int = 0): Bool } +/** The id of a client source or destination. Used in managers. */ trait HasClientId extends TLBundle { val client_id = UInt(width = tlClientIdBits) } -// Actual TileLink channel bundle definitions +/** TileLink channel bundle definitions */ +/** The Acquire channel is used to intiate coherence protocol transactions in + * order to gain access to a cache block's data with certain permissions + * enabled. Messages sent over this channel may be custom types defined by + * a [[uncore.CoherencePolicy]] for cached data accesse or may be built-in types + * used for uncached data accesses. Acquires may contain data for Put or + * PutAtomic built-in types. After sending an Acquire, clients must + * wait for a manager to send them a [[uncore.Grant]] message in response. + */ class Acquire extends ClientToManagerChannel with HasCacheBlockAddress with HasClientTransactionId with HasTileLinkData { - // Actual bundle fields + // Actual bundle fields: val is_builtin_type = Bool() val a_type = UInt(width = tlAcquireTypeBits) val union = Bits(width = tlAcquireUnionBits) - // Utility funcs for accessing subblock union + // Utility funcs for accessing subblock union: val opCodeOff = 1 val opSizeOff = tlMemoryOpcodeBits + opCodeOff val addrByteOff = tlMemoryOperandSizeBits + opSizeOff val addrByteMSB = tlByteAddrBits + addrByteOff + /** Hint whether to allocate the block in any interveneing caches */ def allocate(dummy: Int = 0) = union(0) - def op_code(dummy: Int = 0) = Mux(isBuiltInType(Acquire.putType) || isBuiltInType(Acquire.putBlockType), + /** Op code for [[uncore.PutAtomic]] operations */ + def op_code(dummy: Int = 0) = Mux( + isBuiltInType(Acquire.putType) || isBuiltInType(Acquire.putBlockType), M_XWR, union(opSizeOff-1, opCodeOff)) + /** Operand size for [[uncore.PutAtomic]] */ def op_size(dummy: Int = 0) = union(addrByteOff-1, opSizeOff) + /** Byte address for [[uncore.PutAtomic]] operand */ def addr_byte(dummy: Int = 0) = union(addrByteMSB-1, addrByteOff) private def amo_offset(dummy: Int = 0) = addr_byte()(tlByteAddrBits-1, log2Up(amoAluOperandBits/8)) + /** Bit offset of [[uncore.PutAtomic]] operand */ def amo_shift_bits(dummy: Int = 0) = UInt(amoAluOperandBits)*amo_offset() + /** Write mask for [[uncore.Put]], [[uncore.PutBlock]], [[uncore.PutAtomic]] */ def wmask(dummy: Int = 0) = Mux(isBuiltInType(Acquire.putAtomicType), FillInterleaved(amoAluOperandBits/8, UIntToOH(amo_offset())), Mux(isBuiltInType(Acquire.putBlockType) || isBuiltInType(Acquire.putType), union(tlWriteMaskBits, 1), UInt(0, width = tlWriteMaskBits))) + /** Full, beat-sized writemask */ def full_wmask(dummy: Int = 0) = FillInterleaved(8, wmask()) - + /** Complete physical address for block, beat or operand */ def addr(dummy: Int = 0) = Cat(this.addr_block, this.addr_beat, this.addr_byte()) - // Other helper funcs + // Other helper functions: + /** Message type equality */ def is(t: UInt) = a_type === t //TODO: make this more opaque; def ===? + /** Is this message a built-in or custom type */ def isBuiltInType(dummy: Int = 0): Bool = is_builtin_type + /** Is this message a particular built-in type */ def isBuiltInType(t: UInt): Bool = is_builtin_type && a_type === t + /** Does this message refer to subblock operands using info in the Acquire.union subbundle */ def isSubBlockType(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesOnSubBlocks.contains(a_type) + /** Is this message a built-in prefetch message */ def isPrefetch(dummy: Int = 0): Bool = isBuiltInType() && is(Acquire.prefetchType) - // Assumes no custom types have data + /** Does this message contain data? Assumes that no custom message types have data. */ def hasData(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesWithData.contains(a_type) + /** Does this message contain multiple beats of data? Assumes that no custom message types have data. */ def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && isBuiltInType() && Acquire.typesWithMultibeatData.contains(a_type) + /** Does this message require the manager to probe the client the very client that sent it? + * Needed if multiple caches are attached to the same port. + */ def requiresSelfProbe(dummy: Int = 0) = Bool(false) + /** Mapping between each built-in Acquire type (defined in companion object) + * and a built-in Grant type. + */ def getBuiltInGrantType(dummy: Int = 0): UInt = { MuxLookup(this.a_type, Grant.putAckType, Array( Acquire.getType -> Grant.getDataBeatType, @@ -166,15 +215,33 @@ class Acquire extends ClientToManagerChannel } } +/** [[uncore.Acquire]] with an extra field stating its source id */ +class AcquireFromSrc extends Acquire with HasClientId + +/** Contains definitions of the the built-in Acquire types and a factory + * for [[uncore.Acquire]] + * + * In general you should avoid using this factory directly and use + * [[uncore.ClientMetadata.makeAcquire]] for custom cached Acquires and + * [[uncore.Get]], [[uncore.Put]], etc. for built-in uncached Acquires. + * + * @param is_builtin_type built-in or custom type message? + * @param a_type built-in type enum or custom type enum + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param addr_beat sub-block address (which beat) + * @param data data being put outwards + * @param union additional fields used for uncached types + */ object Acquire { val nBuiltInTypes = 5 //TODO: Use Enum - def getType = UInt("b000") - def getBlockType = UInt("b001") - def putType = UInt("b010") - def putBlockType = UInt("b011") - def putAtomicType = UInt("b100") - def prefetchType = UInt("b101") + def getType = UInt("b000") // Get a single beat of data + def getBlockType = UInt("b001") // Get a whole block of data + def putType = UInt("b010") // Put a single beat of data + def putBlockType = UInt("b011") // Put a whole block of data + def putAtomicType = UInt("b100") // Perform an atomic memory op + def prefetchType = UInt("b101") // Prefetch a whole block of data def typesWithData = Vec(putType, putBlockType, putAtomicType) def typesWithMultibeatData = Vec(putBlockType) def typesOnSubBlocks = Vec(putType, getType, putAtomicType) @@ -208,7 +275,16 @@ object Acquire { } } -// Asks for a single TileLink beat of data +/** Get a single beat of data from the outer memory hierarchy + * + * The client can hint whether he block containing this beat should be + * allocated in the intervening levels of the hierarchy. + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param addr_beat sub-block address (which beat) + * @param alloc hint whether the block should be allocated in intervening caches + */ object Get { def apply( client_xact_id: UInt, @@ -225,7 +301,15 @@ object Get { } } -// Asks for an entire cache block of data +/** Get a whole cache block of data from the outer memory hierarchy + * + * The client can hint whether the block should be allocated in the + * intervening levels of the hierarchy. + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param alloc hint whether the block should be allocated in intervening caches + */ object GetBlock { def apply( client_xact_id: UInt = UInt(0), @@ -240,8 +324,12 @@ object GetBlock { } } -// Prefetch a cache block into the next level of the memory hierarchy -// with read permissions +/** Prefetch a cache block into the next-outermost level of the memory hierarchy + * with read permissions. + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + */ object GetPrefetch { def apply( client_xact_id: UInt, @@ -256,7 +344,16 @@ object GetPrefetch { } } -// Writes up to a single TileLink beat of data, using mask +/** Put a single beat of data into the outer memory hierarchy + * + * The block will be allocated in the next-outermost level of the hierarchy. + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param addr_beat sub-block address (which beat) + * @param data data being refilled to the original requestor + * @param wmask per-byte write mask for this beat + */ object Put { def apply( client_xact_id: UInt, @@ -275,7 +372,19 @@ object Put { } } -// Writes an entire cache block of data +/** Put a whole cache block of data into the outer memory hierarchy + * + * If the write mask is not full, the block will be allocated in the + * next-outermost level of the hierarchy. If the write mask is full, the + * client can hint whether the block should be allocated or not. + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param addr_beat sub-block address (which beat of several) + * @param data data being refilled to the original requestor + * @param wmask per-byte write mask for this beat + * @param alloc hint whether the block should be allocated in intervening caches + */ object PutBlock { def apply( client_xact_id: UInt, @@ -309,7 +418,36 @@ object PutBlock { } } -// Performs an atomic operation in the outer memory +/** Prefetch a cache block into the next-outermost level of the memory hierarchy + * with write permissions. + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + */ +object PutPrefetch { + def apply( + client_xact_id: UInt, + addr_block: UInt): Acquire = { + Acquire( + is_builtin_type = Bool(true), + a_type = Acquire.prefetchType, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = UInt(0), + union = Cat(M_XWR, Bool(true))) + } +} + +/** Perform an atomic memory operation in the next-outermost level of the memory hierarchy + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param addr_beat sub-block address (within which beat) + * @param addr_byte sub-block address (which byte) + * @param atomic_opcode {swap, add, xor, and, min, max, minu, maxu} from [[uncore.MemoryOpConstants]] + * @param operand_size {byte, half, word, double} from [[uncore.MemoryOpConstants]] + * @param data source operand data + */ object PutAtomic { def apply( client_xact_id: UInt, @@ -330,22 +468,11 @@ object PutAtomic { } } -// Prefetch a cache block into the next level of the memory hierarchy -// with write permissions -object PutPrefetch { - def apply( - client_xact_id: UInt, - addr_block: UInt): Acquire = { - Acquire( - is_builtin_type = Bool(true), - a_type = Acquire.prefetchType, - client_xact_id = client_xact_id, - addr_block = addr_block, - addr_beat = UInt(0), - union = Cat(M_XWR, Bool(true))) - } -} - +/** The Probe channel is used to force clients to release data or cede permissions + * on a cache block. Clients respond to Probes with [[uncore.Release]] messages. + * The available types of Probes are customized by a particular + * [[uncore.CoherencePolicy]]. + */ class Probe extends ManagerToClientChannel with HasCacheBlockAddress { val p_type = UInt(width = tlCoh.probeTypeWidth) @@ -355,6 +482,18 @@ class Probe extends ManagerToClientChannel def hasMultibeatData(dummy: Int = 0) = Bool(false) } +/** [[uncore.Probe]] with an extra field stating its destination id */ +class ProbeToDst extends Probe with HasClientId + +/** Contains factories for [[uncore.Probe]] and [[uncore.ProbeToDst]] + * + * In general you should avoid using these factories directly and use + * [[uncore.ManagerMetadata.makeProbe]] instead. + * + * @param dst id of client to which probe should be sent + * @param p_type custom probe type + * @param addr_block address of the cache block + */ object Probe { def apply(p_type: UInt, addr_block: UInt): Probe = { val prb = new Probe @@ -371,6 +510,13 @@ object Probe { } } +/** The Release channel is used to release data or permission back to the manager + * in response to [[uncore.Probe]] messages. It can also be used to voluntarily + * write back data, for example in the event that dirty data must be evicted on + * a cache miss. The available types of Release messages are always customized by + * a particular [[uncore.CoherencePolicy]]. Releases may contain data or may be + * simple acknowledgements. Voluntary Releases are acknowledged with [[uncore.Grants]]. + */ class Release extends ClientToManagerChannel with HasCacheBlockAddress with HasClientTransactionId @@ -387,6 +533,21 @@ class Release extends ClientToManagerChannel def requiresAck(dummy: Int = 0) = !Bool(tlNetworkPreservesPointToPointOrdering) } +/** [[uncore.Release]] with an extra field stating its source id */ +class ReleaseFromSrc extends Release with HasClientId + +/** Contains a [[uncore.Release]] factory + * + * In general you should avoid using this factory directly and use + * [[uncore.ClientMetadata.makeRelease]] instead. + * + * @param voluntary is this a voluntary writeback + * @param r_type type enum defined by coherence protocol + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param addr_beat beat id of the data + * @param data data being written back + */ object Release { def apply( voluntary: Bool, @@ -406,6 +567,13 @@ object Release { } } +/** The Grant channel is used to refill data or grant permissions requested of the + * manager agent via an [[uncore.Acquire]] message. It is also used to acknowledge + * the receipt of voluntary writeback from clients in the form of [[uncore.Release]] + * messages. There are built-in Grant messages used for Gets and Puts, and + * coherence policies may also define custom Grant types. Grants may contain data + * or may be simple acknowledgements. Grants are responded to with [[uncore.Finish]]. + */ class Grant extends ManagerToClientChannel with HasTileLinkData with HasClientTransactionId @@ -433,13 +601,30 @@ class Grant extends ManagerToClientChannel } } +/** [[uncore.Grant]] with an extra field stating its destination */ +class GrantToDst extends Grant with HasClientId + +/** Contains definitions of the the built-in grant types and factories + * for [[uncore.Grant]] and [[uncore.GrantToDst]] + * + * In general you should avoid using these factories directly and use + * [[uncore.ManagerMetadata.makeGrant]] instead. + * + * @param dst id of client to which grant should be sent + * @param is_builtin_type built-in or custom type message? + * @param g_type built-in type enum or custom type enum + * @param client_xact_id client's transaction id + * @param manager_xact_id manager's transaction id + * @param addr_beat beat id of the data + * @param data data being refilled to the original requestor + */ object Grant { val nBuiltInTypes = 5 - def voluntaryAckType = UInt("b000") - def putAckType = UInt("b001") - def prefetchAckType = UInt("b011") - def getDataBeatType = UInt("b100") - def getDataBlockType = UInt("b101") + def voluntaryAckType = UInt("b000") // For acking Releases + def prefetchAckType = UInt("b001") // For acking any kind of Prefetch + def putAckType = UInt("b011") // For acking any kind of non-prfetch Put + def getDataBeatType = UInt("b100") // Supplying a single beat of Get + def getDataBlockType = UInt("b101") // Supplying all beats of a GetBlock def typesWithData = Vec(getDataBlockType, getDataBeatType) def typesWithMultibeatData= Vec(getDataBlockType) @@ -480,62 +665,69 @@ object Grant { } } +/** The Finish channel is used to provide a global ordering of transactions + * in networks that do not guarantee point-to-point ordering of messages. + * A Finsish message is sent as acknowledgement of receipt of a [[uncore.Grant]]. + * When a Finish message is received, a manager knows it is safe to begin + * processing other transactions that touch the same cache block. + */ class Finish extends ClientToManagerChannel with HasManagerTransactionId { def hasData(dummy: Int = 0) = Bool(false) def hasMultibeatData(dummy: Int = 0) = Bool(false) } -// These subtypes include a field for the source or destination ClientId -class AcquireFromSrc extends Acquire with HasClientId -class ProbeToDst extends Probe with HasClientId -class ReleaseFromSrc extends Release with HasClientId -class GrantToDst extends Grant with HasClientId - -// Complete IO definitions for two types of TileLink clients, including -// networking headers +/** Complete IO definition for incoherent TileLink, including networking headers */ class UncachedTileLinkIO extends TLBundle { val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) } +/** Complete IO definition for coherent TileLink, including networking headers */ class TileLinkIO extends UncachedTileLinkIO { val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip val release = new DecoupledIO(new LogicalNetworkIO(new Release)) } -// Converts UncachedTileLinkIO to regular TileLinkIO by pinning -// probe.ready and release.valid low -class TileLinkIOWrapper extends TLModule { - val io = new Bundle { - val in = new UncachedTileLinkIO().flip - val out = new TileLinkIO - } - io.out.acquire <> io.in.acquire - io.out.grant <> io.in.grant - io.out.finish <> io.in.finish - io.out.probe.ready := Bool(true) - io.out.release.valid := Bool(false) -} - -// This version of TileLinkIO does not contain network headers. The headers -// are provided in the top-level that instantiates the clients and network, -// probably using a TileLinkClientPort module. -// By eliding the header subbundles within the clients we can enable -// hierarchical P&R while minimizing unconnected port errors in GDS. -// Secondly, this version of the interface elides Finish messages, with the -// assumption that a FinishUnit has been coupled to the TileLinkIO port -// to deal with acking received Grants. +/** This version of UncachedTileLinkIO does not contain network headers. + * It is intended for use within client agents. + * + * Headers are provided in the top-level that instantiates the clients and network, + * probably using a [[uncore.ClientTileLinkPort]] module. + * By eliding the header subbundles within the clients we can enable + * hierarchical P-and-R while minimizing unconnected port errors in GDS. + * + * Secondly, this version of the interface elides [[uncore.Finish]] messages, with the + * assumption that a [[uncore.FinishUnit]] has been coupled to the TileLinkIO port + * to deal with acking received [[uncore.Grants]]. + */ class ClientUncachedTileLinkIO extends TLBundle { val acquire = new DecoupledIO(new Acquire) val grant = new DecoupledIO(new Grant).flip } +/** This version of TileLinkIO does not contain network headers. + * It is intended for use within client agents. + */ class ClientTileLinkIO extends ClientUncachedTileLinkIO { val probe = new DecoupledIO(new Probe).flip val release = new DecoupledIO(new Release) } +/** This version of TileLinkIO does not contain network headers, but + * every channel does include an extra client_id subbundle. + * It is intended for use within Management agents. + * + * Managers need to track where [[uncore.Acquire]] and [[uncore.Release]] messages + * originated so that they can send a [[uncore.Grant]] to the right place. + * Similarly they must be able to issues Probes to particular clients. + * However, we'd still prefer to have [[uncore.ManagerTileLinkPort]] fill in + * the header.src to enable hierarchical p-and-r of the managers. Additionally, + * coherent clients might be mapped to random network port ids, and we'll leave it to the + * [[uncore.ManagerTileLinkPort]] to apply the correct mapping. Managers do need to + * see Finished so they know when to allow new transactions on a cache + * block to proceed. + */ class ManagerTileLinkIO extends TLBundle { val acquire = new DecoupledIO(new AcquireFromSrc).flip val grant = new DecoupledIO(new GrantToDst) @@ -544,17 +736,7 @@ class ManagerTileLinkIO extends TLBundle { val release = new DecoupledIO(new ReleaseFromSrc).flip } -class ClientTileLinkIOWrapper extends TLModule { - val io = new Bundle { - val in = new ClientUncachedTileLinkIO().flip - val out = new ClientTileLinkIO - } - io.out.acquire <> io.in.acquire - io.out.grant <> io.in.grant - io.out.probe.ready := Bool(true) - io.out.release.valid := Bool(false) -} - +/** Utilities for safely wrapping a *UncachedTileLink by pinning probe.ready and release.valid low */ object TileLinkIOWrapper { def apply(utl: ClientUncachedTileLinkIO, p: Parameters): ClientTileLinkIO = { val conv = Module(new ClientTileLinkIOWrapper)(p) @@ -580,13 +762,32 @@ object TileLinkIOWrapper { def apply(tl: TileLinkIO): TileLinkIO = tl } -class FinishQueueEntry extends TLBundle { - val fin = new Finish - val dst = UInt(width = log2Up(params(LNEndpoints))) +class TileLinkIOWrapper extends TLModule { + val io = new Bundle { + val in = new UncachedTileLinkIO().flip + val out = new TileLinkIO + } + io.out.acquire <> io.in.acquire + io.out.grant <> io.in.grant + io.out.finish <> io.in.finish + io.out.probe.ready := Bool(true) + io.out.release.valid := Bool(false) } -class FinishQueue(entries: Int) extends Queue(new FinishQueueEntry, entries) +class ClientTileLinkIOWrapper extends TLModule { + val io = new Bundle { + val in = new ClientUncachedTileLinkIO().flip + val out = new ClientTileLinkIO + } + io.out.acquire <> io.in.acquire + io.out.grant <> io.in.grant + io.out.probe.ready := Bool(true) + io.out.release.valid := Bool(false) +} +/** A helper module that automatically issues [[uncore.Finish]] messages in repsonse + * to [[uncore.Grant]] that it receives from a manager and forwards to a client + */ class FinishUnit(srcId: Int = 0, outstanding: Int = 2) extends TLModule with HasDataBeatCounters { val io = new Bundle { val grant = Decoupled(new LogicalNetworkIO(new Grant)).flip @@ -633,36 +834,23 @@ class FinishUnit(srcId: Int = 0, outstanding: Int = 2) extends TLModule with Has } } -object ClientTileLinkHeaderCreator { - def apply[T <: ClientToManagerChannel with HasCacheBlockAddress : ClassTag]( - in: DecoupledIO[T], - clientId: Int, - addrConvert: UInt => UInt): DecoupledIO[LogicalNetworkIO[T]] = { - val out = new DecoupledIO(new LogicalNetworkIO(in.bits.clone)).asDirectionless - out.bits.payload := in.bits - out.bits.header.src := UInt(clientId) - out.bits.header.dst := addrConvert(in.bits.addr_block) - out.valid := in.valid - in.ready := out.ready - out - } +class FinishQueueEntry extends TLBundle { + val fin = new Finish + val dst = UInt(width = log2Up(params(LNEndpoints))) } -object ManagerTileLinkHeaderCreator { - def apply[T <: ManagerToClientChannel with HasClientId : ClassTag]( - in: DecoupledIO[T], - managerId: Int, - idConvert: UInt => UInt): DecoupledIO[LogicalNetworkIO[T]] = { - val out = new DecoupledIO(new LogicalNetworkIO(in.bits.clone)).asDirectionless - out.bits.payload := in.bits - out.bits.header.src := UInt(managerId) - out.bits.header.dst := idConvert(in.bits.client_id) - out.valid := in.valid - in.ready := out.ready - out - } -} +class FinishQueue(entries: Int) extends Queue(new FinishQueueEntry, entries) +/** A port to convert [[uncore.ClientTileLinkIO]].flip into [[uncore.TileLinkIO]] + * + * Creates network headers for [[uncore.Acquire]] and [[uncore.Release]] messages, + * calculating header.dst and filling in header.src. + * Strips headers from [[uncore.Probes]]. + * Responds to [[uncore.Grant]] by automatically issuing [[uncore.Finish]] to the granting managers. + * + * @param clientId network port id of this agent + * @param addrConvert how a physical address maps to a destination manager port id + */ class ClientTileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) extends TLModule { val io = new Bundle { val client = new ClientTileLinkIO().flip @@ -686,6 +874,31 @@ class ClientTileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) extend io.client.grant <> gnt_without_header } +object ClientTileLinkHeaderCreator { + def apply[T <: ClientToManagerChannel with HasCacheBlockAddress]( + in: DecoupledIO[T], + clientId: Int, + addrConvert: UInt => UInt): DecoupledIO[LogicalNetworkIO[T]] = { + val out = new DecoupledIO(new LogicalNetworkIO(in.bits.clone)).asDirectionless + out.bits.payload := in.bits + out.bits.header.src := UInt(clientId) + out.bits.header.dst := addrConvert(in.bits.addr_block) + out.valid := in.valid + in.ready := out.ready + out + } +} + +/** A port to convert [[uncore.ManagerTileLinkIO]].flip into [[uncore.TileLinkIO]].flip + * + * Creates network headers for [[uncore.Probe]] and [[uncore.Grant]] messagess, + * calculating header.dst and filling in header.src. + * Strips headers from [[uncore.Acquire]], [[uncore.Release]] and [[uncore.Finish], + * but supplies client_id instead. + * + * @param managerId the network port id of this agent + * @param idConvert how a sharer id maps to a destination client port id + */ class ManagerTileLinkNetworkPort(managerId: Int, idConvert: UInt => UInt) extends TLModule { val io = new Bundle { val manager = new ManagerTileLinkIO().flip @@ -700,8 +913,25 @@ class ManagerTileLinkNetworkPort(managerId: Int, idConvert: UInt => UInt) extend io.manager.finish <> DecoupledLogicalNetworkIOUnwrapper(io.network.finish) } +object ManagerTileLinkHeaderCreator { + def apply[T <: ManagerToClientChannel with HasClientId]( + in: DecoupledIO[T], + managerId: Int, + idConvert: UInt => UInt): DecoupledIO[LogicalNetworkIO[T]] = { + val out = new DecoupledIO(new LogicalNetworkIO(in.bits.clone)).asDirectionless + out.bits.payload := in.bits + out.bits.header.src := UInt(managerId) + out.bits.header.dst := idConvert(in.bits.client_id) + out.valid := in.valid + in.ready := out.ready + out + } +} + +/** Struct for describing per-channel queue depths */ case class TileLinkDepths(acq: Int, prb: Int, rel: Int, gnt: Int, fin: Int) +/** Optionally enqueues each [[uncore.TileLinkChannel]] individually */ class TileLinkEnqueuer(depths: TileLinkDepths) extends Module { val io = new Bundle { val client = new TileLinkIO().flip @@ -905,10 +1135,10 @@ class ClientTileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiter } /** Utility trait containing wiring functions to keep track of how many data beats have - * been sent or recieved over a particular TileLinkChannel or pair of channels. + * been sent or recieved over a particular [[uncore.TileLinkChannel]] or pair of channels. * * Won't count message types that don't have data. - * Used in XactTrackers and FinishUnit. + * Used in [[uncore.XactTracker]] and [[uncore.FinishUnit]]. */ trait HasDataBeatCounters { type HasBeat = TileLinkChannel with HasTileLinkBeatId @@ -926,7 +1156,7 @@ trait HasDataBeatCounters { (cnt, done) } - /** Counter for beats on outgoing DecoupledIOs */ + /** Counter for beats on outgoing [[chisel.DecoupledIO]] */ def connectOutgoingDataBeatCounter[T <: TileLinkChannel](in: DecoupledIO[T], beat: UInt = UInt(0)): (UInt, Bool) = connectDataBeatCounter(in.fire(), in.bits, beat) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 8dda15b7..d7573aec 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -2,15 +2,10 @@ package uncore import Chisel._ -import scala.reflect._ -import scala.reflect.runtime.universe._ case object NReleaseTransactors extends Field[Int] case object NProbeTransactors extends Field[Int] case object NAcquireTransactors extends Field[Int] -case object NIncoherentClients extends Field[Int] -case object NCoherentClients extends Field[Int] -case object L2CoherencePolicy extends Field[CoherencePolicy] trait CoherenceAgentParameters extends UsesParameters { val nReleaseTransactors = 1 @@ -28,6 +23,7 @@ trait CoherenceAgentParameters extends UsesParameters { val innerByteAddrBits = log2Up(innerDataBits/8) require(outerDataBeats == innerDataBeats) //TODO: must fix all xact_data Vecs to remove this requirement } + abstract class CoherenceAgentBundle extends Bundle with CoherenceAgentParameters abstract class CoherenceAgentModule extends Module with CoherenceAgentParameters @@ -53,7 +49,7 @@ trait HasCoherenceAgentWiringHelpers { trait HasInnerTLIO extends CoherenceAgentBundle { val inner = Bundle(new ManagerTileLinkIO)(innerTLParams) - val incoherent = Vec.fill(inner.tlNCoherentClients){Bool()}.asInput + val incoherent = Vec.fill(inner.tlNCachingClients){Bool()}.asInput def iacq(dummy: Int = 0) = inner.acquire.bits def iprb(dummy: Int = 0) = inner.probe.bits def irel(dummy: Int = 0) = inner.release.bits @@ -129,5 +125,5 @@ abstract class XactTracker extends CoherenceAgentModule with HasDataBeatCounters dropPendingBitWhenBeat(in.fire() && in.bits.hasData(), in.bits) def dropPendingBitAtDest(in: DecoupledIO[ProbeToDst]): UInt = - ~Fill(in.bits.tlNCoherentClients, in.fire()) | ~UIntToOH(in.bits.client_id) + ~Fill(in.bits.tlNCachingClients, in.fire()) | ~UIntToOH(in.bits.client_id) } From f66a9fd7a6fd8f438c823cae85b24fd7cc4cd0c7 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 20 Apr 2015 10:46:02 -0700 Subject: [PATCH 369/374] simplify ClientMetadata.makeRelease --- uncore/src/main/scala/metadata.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/uncore/src/main/scala/metadata.scala b/uncore/src/main/scala/metadata.scala index cf52d286..f91035f4 100644 --- a/uncore/src/main/scala/metadata.scala +++ b/uncore/src/main/scala/metadata.scala @@ -59,13 +59,12 @@ class ClientMetadata extends CoherenceMetadata { def makeRelease( prb: Probe, - client_xact_id: UInt, addr_beat: UInt = UInt(0), data: UInt = UInt(0)): Release = { Bundle(Release( voluntary = Bool(false), r_type = co.getReleaseType(prb, this), - client_xact_id = client_xact_id, + client_xact_id = UInt(0), addr_block = prb.addr_block, addr_beat = addr_beat, data = data), { case TLId => id }) From 4c7969b2b37bafa83dac60b1a20c6288110d958b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 20 Apr 2015 16:32:09 -0700 Subject: [PATCH 370/374] Metadata docs and api cleanup --- uncore/src/main/scala/cache.scala | 25 ++-- uncore/src/main/scala/metadata.scala | 173 ++++++++++++++++++++++----- uncore/src/main/scala/tilelink.scala | 7 ++ 3 files changed, 160 insertions(+), 45 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 75bb9e1f..4adb09a3 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -338,10 +338,6 @@ class L2DataArray(delay: Int) extends L2HellaCacheModule { io.write.ready := Bool(true) } -class L2SecondaryMissInfo extends TLBundle - with HasTileLinkBeatId - with HasClientTransactionId - class L2HellaCacheBank extends HierarchicalCoherenceAgent with L2HellaCacheParameters { require(isPow2(nSets)) @@ -617,10 +613,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { val pending_coh = Reg{ xact_old_meta.coh.clone } // Secondary miss queue - val ignt_q = Module(new Queue(new L2SecondaryMissInfo, nSecondaryMisses))(innerTLParams) - ignt_q.io.enq.bits.client_xact_id := io.iacq().client_xact_id - ignt_q.io.enq.bits.addr_beat := io.iacq().addr_beat - // TODO add ignt.dst <- iacq.src + val ignt_q = Module(new Queue(new SecondaryMissInfo, nSecondaryMisses))(innerTLParams) // State holding progress made on processing this transaction val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) @@ -741,6 +734,12 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { can_merge_iacq_put || can_merge_iacq_get + // Enqueue secondary miss information + ignt_q.io.enq.valid := iacq_data_done + ignt_q.io.enq.bits.client_xact_id := io.iacq().client_xact_id + ignt_q.io.enq.bits.addr_beat := io.iacq().addr_beat + // TODO add ignt.dst <- iacq.src + // Track whether any beats are missing from a PutBlock pending_puts := (pending_puts & dropPendingBitWhenBeatHasData(io.inner.acquire)) @@ -792,22 +791,20 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { addPendingBitWhenBeatHasData(io.inner.release) | addPendingBitWhenBeatHasData(io.outer.grant) | addPendingBitInternal(io.data.resp) - ignt_q.io.enq.valid := iacq_data_done ignt_q.io.deq.ready := ignt_data_done - // Make the Grant message using the data stored in the secondary miss queue io.inner.grant.valid := state === s_busy && ignt_q.io.deq.valid && (!io.ignt().hasData() || pending_ignt_data(ignt_data_idx)) + // Make the Grant message using the data stored in the secondary miss queue io.inner.grant.bits := pending_coh.inner.makeGrant( - dst = xact.client_id, // TODO: ignt_q.io.deq.bits.src - acq = xact, - client_xact_id = ignt_q.io.deq.bits.client_xact_id, + pri = xact, + sec = ignt_q.io.deq.bits, manager_xact_id = UInt(trackerId), - addr_beat = ignt_data_idx, data = Mux(xact.is(Acquire.putAtomicType), amo_result, data_buffer(ignt_data_idx))) + io.inner.grant.bits.addr_beat := ignt_data_idx // override based on outgoing counter val pending_coh_on_ignt = HierarchicalMetadata( pending_coh.inner.onGrant(io.ignt()), diff --git a/uncore/src/main/scala/metadata.scala b/uncore/src/main/scala/metadata.scala index f91035f4..74e410c6 100644 --- a/uncore/src/main/scala/metadata.scala +++ b/uncore/src/main/scala/metadata.scala @@ -3,7 +3,7 @@ package uncore import Chisel._ -// Classes to represent coherence information in clients and managers +/** Base class to represent coherence information in clients and managers */ abstract class CoherenceMetadata extends Bundle { val co = params(TLCoherencePolicy) val id = params(TLId) @@ -15,21 +15,35 @@ abstract class CoherenceMetadata extends Bundle { * memory operations or [[uncore.Probe]] messages. */ class ClientMetadata extends CoherenceMetadata { + /** Actual state information stored in this bundle */ val state = UInt(width = co.clientStateWidth) + /** Metadata equality */ def ===(rhs: ClientMetadata): Bool = this.state === rhs.state def !=(rhs: ClientMetadata): Bool = !this.===(rhs) + /** Is the block's data present in this cache */ def isValid(dummy: Int = 0): Bool = co.isValid(this) - def isHit(cmd: UInt): Bool = co.isHit(cmd, this) - def isMiss(cmd: UInt): Bool = !co.isHit(cmd, this) - def requiresAcquireOnSecondaryMiss(first_cmd: UInt, second_cmd: UInt): Bool = - co.requiresAcquireOnSecondaryMiss(first_cmd, second_cmd, this) - def requiresReleaseOnCacheControl(cmd: UInt): Bool = - co.requiresReleaseOnCacheControl(cmd: UInt, this) + /** Does this cache have permissions on this block sufficient to perform op */ + def isHit(op_code: UInt): Bool = co.isHit(op_code, this) + /** Does this cache lack permissions on this block sufficient to perform op */ + def isMiss(op_code: UInt): Bool = !co.isHit(op_code, this) + /** Does a secondary miss on the block require another Acquire message */ + def requiresAcquireOnSecondaryMiss(first_op: UInt, second_op: UInt): Bool = + co.requiresAcquireOnSecondaryMiss(first_op, second_op, this) + /** Does op require a Release to be made to outer memory */ + def requiresReleaseOnCacheControl(op_code: UInt): Bool = + co.requiresReleaseOnCacheControl(op_code: UInt, this) + /** Does an eviction require a Release to be made to outer memory */ def requiresVoluntaryWriteback(dummy: Int = 0): Bool = co.requiresReleaseOnCacheControl(M_FLUSH, this) + /** Constructs an Acquire message based on this metdata and a memory operation + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param op_code a memory operation from [[uncore.MemoryOpConstants]] + */ def makeAcquire( client_xact_id: UInt, addr_block: UInt, @@ -43,6 +57,13 @@ class ClientMetadata extends CoherenceMetadata { { case TLId => id }) } + /** Constructs a Release message based on this metadata on an eviction + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param addr_beat sub-block address (which beat) + * @param data data being written back + */ def makeVoluntaryWriteback( client_xact_id: UInt, addr_block: UInt, @@ -57,6 +78,12 @@ class ClientMetadata extends CoherenceMetadata { data = data), { case TLId => id }) } + /** Constructs a Release message based on this metadata and a [[uncore.Probe]] + * + * @param the incoming [[uncore.Probe]] + * @param addr_beat sub-block address (which beat) + * @param data data being released + */ def makeRelease( prb: Probe, addr_beat: UInt = UInt(0), @@ -70,16 +97,37 @@ class ClientMetadata extends CoherenceMetadata { data = data), { case TLId => id }) } + /** New metadata after receiving a [[uncore.Grant]] + * + * @param incoming the incoming [[uncore.Grant]] + * @param pending the mem op that triggered this transaction + */ def onGrant(incoming: Grant, pending: UInt): ClientMetadata = Bundle(co.clientMetadataOnGrant(incoming, pending, this), { case TLId => id }) + + /** New metadata after receiving a [[uncore.Probe]] + * + * @param incoming the incoming [[uncore.Probe]] + */ def onProbe(incoming: Probe): ClientMetadata = Bundle(co.clientMetadataOnProbe(incoming, this), { case TLId => id }) - def onHit(cmd: UInt): ClientMetadata = - Bundle(co.clientMetadataOnHit(cmd, this), { case TLId => id }) - def onCacheControl(cmd: UInt): ClientMetadata = - Bundle(co.clientMetadataOnCacheControl(cmd, this), { case TLId => id }) + + /** New metadata after a op_code hits this block + * + * @param op_code a memory operation from [[uncore.MemoryOpConstants]] + */ + def onHit(op_code: UInt): ClientMetadata = + Bundle(co.clientMetadataOnHit(op_code, this), { case TLId => id }) + + /** New metadata after receiving a [[uncore.Probe]] + * + * @param op_code a memory operation from [[uncore.MemoryOpConstants]] + */ + def onCacheControl(op_code: UInt): ClientMetadata = + Bundle(co.clientMetadataOnCacheControl(op_code, this), { case TLId => id }) } +/** Factories for ClientMetadata, including on reset */ object ClientMetadata { def apply(state: UInt) = { val meta = new ClientMetadata @@ -89,33 +137,64 @@ object ClientMetadata { def onReset = new ClientMetadata().co.clientMetadataOnReset } -/* The ManagerMetadata stores manager-side information about the status - of a cache block, including whether it has any known sharers. Its - API can be used to create Probe and Grant TileLink messages. -*/ +/** Stores manager-side information about the status + * of a cache block, including whether it has any known sharers. + * + * Its API can be used to create [[uncore.Probe]] and [[uncore.Grant]] messages. + */ class ManagerMetadata extends CoherenceMetadata { + // Currently no coherence policies assume manager-side state information // val state = UInt(width = co.masterStateWidth) TODO: Fix 0-width wires in Chisel + + /** The directory information for this block */ val sharers = UInt(width = co.dir.width) + /** Metadata equality */ def ===(rhs: ManagerMetadata): Bool = //this.state === rhs.state && TODO: Fix 0-width wires in Chisel this.sharers === rhs.sharers def !=(rhs: ManagerMetadata): Bool = !this.===(rhs) - def full(dummy: Int = 0) = co.dir.full(this.sharers) + /** Converts the directory info into an N-hot sharer bitvector (i.e. full representation) */ + def full(dummy: Int = 0): UInt = co.dir.full(this.sharers) + + /** Does this [[uncore.Acquire]] require [[uncore.Probes]] to be sent */ def requiresProbes(acq: Acquire): Bool = co.requiresProbes(acq, this) - def requiresProbes(cmd: UInt): Bool = co.requiresProbes(cmd, this) + /** Does this memory op require [[uncore.Probes]] to be sent */ + def requiresProbes(op_code: UInt): Bool = co.requiresProbes(op_code, this) + /** Does an eviction require [[uncore.Probes]] to be sent */ def requiresProbesOnVoluntaryWriteback(dummy: Int = 0): Bool = co.requiresProbes(M_FLUSH, this) - def makeProbe(dst: UInt, cmd: UInt, addr_block: UInt): ProbeToDst = - Bundle(Probe(dst, co.getProbeType(cmd, this), addr_block), { case TLId => id }) - + /** Construct an appropriate [[uncore.ProbeToDst]] for a given [[uncore.Acquire]] + * + * @param dst Destination client id for this Probe + * @param acq Acquire message triggering this Probe + */ def makeProbe(dst: UInt, acq: Acquire): ProbeToDst = Bundle(Probe(dst, co.getProbeType(acq, this), acq.addr_block), { case TLId => id }) + /** Construct an appropriate [[uncore.ProbeToDst]] for a given mem op + * + * @param dst Destination client id for this Probe + * @param op_code memory operation triggering this Probe + * @param addr_block address of the cache block being probed + */ + def makeProbe(dst: UInt, op_code: UInt, addr_block: UInt): ProbeToDst = + Bundle(Probe(dst, co.getProbeType(op_code, this), addr_block), { case TLId => id }) + + /** Construct an appropriate [[uncore.ProbeToDst]] for an eviction + * + * @param dst Destination client id for this Probe + * @param addr_block address of the cache block being probed prior to eviction + */ def makeProbeForVoluntaryWriteback(dst: UInt, addr_block: UInt): ProbeToDst = makeProbe(dst, M_FLUSH, addr_block) + /** Construct an appropriate [[uncore.GrantToDst]] to acknowledge an [[uncore.Release]] + * + * @param rel Release message being acknowledged by this Grant + * @param manager_xact_id manager's transaction id + */ def makeGrant(rel: ReleaseFromSrc, manager_xact_id: UInt): GrantToDst = { Bundle(Grant( dst = rel.client_id, @@ -125,6 +204,15 @@ class ManagerMetadata extends CoherenceMetadata { manager_xact_id = manager_xact_id), { case TLId => id }) } + /** Construct an appropriate [[uncore.GrantToDst]] to respond to an [[uncore.Acquire]] + * + * May contain single or multiple beats of data, or just be a permissions upgrade. + * + * @param acq Acquire message being responded to by this Grant + * @param manager_xact_id manager's transaction id + * @param addr_beat beat id of the data + * @param data data being refilled to the original requestor + */ def makeGrant( acq: AcquireFromSrc, manager_xact_id: UInt, @@ -142,26 +230,42 @@ class ManagerMetadata extends CoherenceMetadata { data = data), { case TLId => id }) } + /** Construct an [[uncore.GrantToDst]] to respond to an [[uncore.Acquire]] with some overrides + * + * Used to respond to secondary misses merged into this transaction. + * May contain single or multiple beats of data. + * + * @param pri Primary miss's Acquire message, used to get g_type and dst + * @param sec Secondary miss info, used to get beat and client_xact_id + * @param manager_xact_id manager's transaction id + * @param data data being refilled to the original requestor + */ def makeGrant( - dst: UInt, - acq: AcquireFromSrc, - client_xact_id: UInt, + pri: AcquireFromSrc, + sec: SecondaryMissInfo, manager_xact_id: UInt, - addr_beat: UInt, data: UInt): GrantToDst = { - val g = makeGrant(acq, manager_xact_id, addr_beat, data) - g.client_id := dst - g.client_xact_id := client_xact_id + val g = makeGrant(pri, manager_xact_id, sec.addr_beat, data) + g.client_xact_id := sec.client_xact_id g } + /** New metadata after receiving a [[uncore.ReleaseFromSrc]] + * + * @param incoming the incoming [[uncore.ReleaseFromSrc]] + */ def onRelease(incoming: ReleaseFromSrc): ManagerMetadata = Bundle(co.managerMetadataOnRelease(incoming, incoming.client_id, this), { case TLId => id }) + /** New metadata after sending a [[uncore.GrantToDst]] + * + * @param outgoing the outgoing [[uncore.GrantToDst]] + */ def onGrant(outgoing: GrantToDst): ManagerMetadata = Bundle(co.managerMetadataOnGrant(outgoing, outgoing.client_id, this), { case TLId => id }) } +/** Factories for ManagerMetadata, including on reset */ object ManagerMetadata { def apply(sharers: UInt, state: UInt = UInt(width = 0)) = { val meta = new ManagerMetadata @@ -178,10 +282,14 @@ object ManagerMetadata { def onReset = new ManagerMetadata().co.managerMetadataOnReset } -/* HierarchicalMetadata is used in a cache in a multi-level memory hierarchy - that is a Manager with respect to some inner caches and a Client with - respect to some outer cache. -*/ +/** HierarchicalMetadata is used in a cache in a multi-level memory hierarchy + * that is a manager with respect to some inner caches and a client with + * respect to some outer cache. + * + * This class makes use of two different sets of TileLink parameters, which are + * applied by contextually mapping [[uncore.TLId]] to one of + * [[uncore.InnerTLId]] or [[uncore.OuterTLId]]. + */ class HierarchicalMetadata extends CoherenceMetadata { val inner: ManagerMetadata = Bundle(new ManagerMetadata, {case TLId => params(InnerTLId)}) val outer: ClientMetadata = Bundle(new ClientMetadata, {case TLId => params(OuterTLId)}) @@ -190,6 +298,7 @@ class HierarchicalMetadata extends CoherenceMetadata { def !=(rhs: HierarchicalMetadata): Bool = !this.===(rhs) } +/** Factories for HierarchicalMetadata, including on reset */ object HierarchicalMetadata { def apply(inner: ManagerMetadata, outer: ClientMetadata): HierarchicalMetadata = { val m = new HierarchicalMetadata @@ -200,5 +309,7 @@ object HierarchicalMetadata { def onReset: HierarchicalMetadata = apply(ManagerMetadata.onReset, ClientMetadata.onReset) } +/** Identifies the TLId of the inner network in a hierarchical cache controller */ case object InnerTLId extends Field[String] +/** Identifies the TLId of the outer network in a hierarchical cache controller */ case object OuterTLId extends Field[String] diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index b759d48b..1c1c2bb4 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -785,6 +785,13 @@ class ClientTileLinkIOWrapper extends TLModule { io.out.release.valid := Bool(false) } +/** Used to track metadata for transactions where multiple secondary misses have been merged + * and handled by a single transaction tracker. + */ +class SecondaryMissInfo extends TLBundle // TODO: add a_type to merge e.g. Get+GetBlocks, and/or HasClientId + with HasTileLinkBeatId + with HasClientTransactionId + /** A helper module that automatically issues [[uncore.Finish]] messages in repsonse * to [[uncore.Grant]] that it receives from a manager and forwards to a client */ From 11b5222d014a425ac65545491e92800b69c01336 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 21 Apr 2015 22:23:04 -0700 Subject: [PATCH 371/374] Refactored WritebackUnit --- uncore/src/main/scala/cache.scala | 245 +++++++++++++----------------- 1 file changed, 105 insertions(+), 140 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 4adb09a3..32529202 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -192,9 +192,8 @@ abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters { } def doInternalInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { - val idx = in.bits.id outs.map(_.bits := in.bits) - outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && idx === UInt(i) } + outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && in.bits.id === UInt(i) } } } @@ -621,7 +620,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { max = io.inner.tlNCachingClients, up = io.inner.probe, down = io.inner.release)._1 - val (pending_ognts, oacq_data_idx, oacq_data_done, ognt_data_idx, ognt_data_done) = + val (pending_ognt, oacq_data_idx, oacq_data_done, ognt_data_idx, ognt_data_done) = connectTwoWayBeatCounter( max = 1, up = io.outer.acquire, @@ -646,7 +645,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { pending_writes.orR || pending_resps.orR || pending_puts.orR || - pending_ognts || + pending_ognt || ignt_q.io.count > UInt(0) || //pending_meta_write || // Has own state: s_meta_write pending_ifins) @@ -794,8 +793,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { ignt_q.io.deq.ready := ignt_data_done io.inner.grant.valid := state === s_busy && ignt_q.io.deq.valid && - (!io.ignt().hasData() || - pending_ignt_data(ignt_data_idx)) + (!io.ignt().hasData() || pending_ignt_data(ignt_data_idx)) // Make the Grant message using the data stored in the secondary miss queue io.inner.grant.bits := pending_coh.inner.makeGrant( pri = xact, @@ -826,9 +824,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire) | addPendingBitWhenBeatHasPartialWritemask(io.inner.acquire) val curr_read_beat = PriorityEncoder(pending_reads) - io.data.read.valid := state === s_busy && - pending_reads.orR && - !pending_ognts + io.data.read.valid := state === s_busy && pending_reads.orR && !pending_ognt io.data.read.bits.id := UInt(trackerId) io.data.read.bits.way_en := xact_way_en io.data.read.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) @@ -847,7 +843,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { val curr_write_beat = PriorityEncoder(pending_writes) io.data.write.valid := state === s_busy && pending_writes.orR && - !pending_ognts && + !pending_ognt && !pending_reads(curr_write_beat) && !pending_resps(curr_write_beat) io.data.write.bits.id := UInt(trackerId) @@ -870,10 +866,11 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { io.meta.write.bits.data.coh := pending_coh io.wb.req.valid := state === s_wb_req - io.wb.req.bits.addr_block := Cat(xact_old_meta.tag, xact.addr_block(idxMSB,idxLSB)) + io.wb.req.bits.id := UInt(trackerId) + io.wb.req.bits.idx := xact.addr_block(idxMSB,idxLSB) + io.wb.req.bits.tag := xact_old_meta.tag io.wb.req.bits.coh := xact_old_meta.coh io.wb.req.bits.way_en := xact_way_en - io.wb.req.bits.id := UInt(trackerId) // Handling of secondary misses (Gets and Puts only for now) when(io.inner.acquire.fire() && io.iacq().hasData()) { // state <= s_meta_wrtie @@ -961,9 +958,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { // These IOs are used for routing in the parent val in_same_set = xact.addr_block(idxMSB,idxLSB) === io.iacq().addr_block(idxMSB,idxLSB) - io.has_release_match := xact.conflicts(io.irel()) && - !io.irel().isVoluntary() && - (state === s_inner_probe) + io.has_release_match := xact.conflicts(io.irel()) && !io.irel().isVoluntary() && io.inner.release.ready io.has_acquire_match := can_merge_iacq_put || can_merge_iacq_get io.has_acquire_conflict := in_same_set && (state != s_idle) && !io.has_acquire_match //TODO: relax from in_same_set to xact.conflicts(io.iacq())? @@ -974,10 +969,8 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { "AcquireTracker accepted data beat from different network source than initial request.") } -class L2WritebackReq extends L2HellaCacheBundle - with HasL2Id { - val addr_block = UInt(width = blockAddrBits) // TODO: assumes same block size - val coh = new HierarchicalMetadata +class L2WritebackReq extends L2Metadata with HasL2Id { + val idx = Bits(width = idxBits) val way_en = Bits(width = nWays) } @@ -996,136 +989,108 @@ class L2WritebackUnitIO extends HierarchicalXactTrackerIO { class L2WritebackUnit(trackerId: Int) extends L2XactTracker { val io = new L2WritebackUnitIO - val s_idle :: s_inner_probe :: s_data_read :: s_data_resp :: s_outer_release :: s_outer_grant :: s_wb_resp :: Nil = Enum(UInt(), 7) + val s_idle :: s_inner_probe :: s_busy :: s_outer_grant :: s_wb_resp :: Nil = Enum(UInt(), 5) val state = Reg(init=s_idle) - val xact_addr_block = Reg(io.wb.req.bits.addr_block.clone) - val xact_coh = Reg{ new HierarchicalMetadata } - val xact_way_en = Reg{ Bits(width = nWays) } - val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) } - val xact_id = Reg{ UInt() } + val xact = Reg(new L2WritebackReq) + val data_buffer = Vec.fill(innerDataBeats){ Reg(init=UInt(0, width = innerDataBits)) } + val xact_addr_block = Cat(xact.tag, xact.idx) - val irel_had_data = Reg(init = Bool(false)) - val irel_cnt = Reg(init = UInt(0, width = log2Up(io.inner.tlNCachingClients+1))) - val pending_probes = Reg(init = Bits(0, width = io.inner.tlNCachingClients)) - val curr_probe_dst = PriorityEncoder(pending_probes) - val full_sharers = io.wb.req.bits.coh.inner.full() - val mask_incoherent = full_sharers & ~io.incoherent.toBits + val pending_irels = + connectTwoWayBeatCounter(max = io.inner.tlNCachingClients, up = io.inner.probe, down = io.inner.release)._1 + val (pending_ognt, orel_data_idx, orel_data_done, ognt_data_idx, ognt_data_done) = + connectTwoWayBeatCounter(max = 1, up = io.outer.release, down = io.outer.grant) + val pending_iprbs = Reg(init = Bits(0, width = io.inner.tlNCachingClients)) + val pending_reads = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_resps = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_orel_data = Reg(init=Bits(0, width = io.inner.tlDataBeats)) - val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val (orel_data_cnt, orel_data_done) = connectOutgoingDataBeatCounter(io.outer.release) - val (read_data_cnt, read_data_done) = connectInternalDataBeatCounter(io.data.read) - val resp_data_done = connectInternalDataBeatCounter(io.data.resp) + // Start the writeback sub-transaction + io.wb.req.ready := state === s_idle - val pending_icoh_on_irel = xact_coh.inner.onRelease(io.irel()) - val pending_ocoh_on_irel = xact_coh.outer.onHit(M_XWR) // WB is a write - val pending_ofin_on_ognt = io.ognt().makeFinish() + // Track which clients yet need to be probed and make Probe message + pending_iprbs := pending_iprbs & dropPendingBitAtDest(io.inner.probe) + val curr_probe_dst = PriorityEncoder(pending_iprbs) + io.inner.probe.valid := state === s_inner_probe && pending_iprbs.orR + io.inner.probe.bits := xact.coh.inner.makeProbeForVoluntaryWriteback(curr_probe_dst, xact_addr_block) - io.has_acquire_conflict := Bool(false) - io.has_acquire_match := Bool(false) - io.has_release_match := !io.irel().isVoluntary() && - io.irel().conflicts(xact_addr_block) && - (state === s_inner_probe) + // Handle incoming releases from clients, which may reduce sharer counts + // and/or write back dirty data + val inner_coh_on_irel = xact.coh.inner.onRelease(io.irel()) + val outer_coh_on_irel = xact.coh.outer.onHit(M_XWR) + io.inner.release.ready := state === s_inner_probe || state === s_busy + when(io.inner.release.fire()) { + xact.coh.inner := inner_coh_on_irel + when(io.irel().hasData()) { xact.coh.outer := outer_coh_on_irel } // WB is a write + data_buffer(io.inner.release.bits.addr_beat) := io.inner.release.bits.data + } - io.outer.acquire.valid := Bool(false) - io.outer.probe.ready := Bool(false) - io.outer.release.valid := Bool(false) // default - io.outer.release.bits := xact_coh.outer.makeVoluntaryWriteback( - client_xact_id = UInt(trackerId), - addr_block = xact_addr_block, - addr_beat = orel_data_cnt, - data = data_buffer(orel_data_cnt)) - io.outer.grant.ready := Bool(false) // default - - io.inner.probe.valid := Bool(false) - io.inner.probe.bits := xact_coh.inner.makeProbeForVoluntaryWriteback(curr_probe_dst, xact_addr_block) - - io.inner.grant.valid := Bool(false) - io.inner.acquire.ready := Bool(false) - io.inner.release.ready := Bool(false) - io.inner.finish.ready := Bool(false) - - io.data.read.valid := Bool(false) + // If a release didn't write back data, have to read it from data array + pending_reads := (pending_reads & + dropPendingBit(io.data.read) & + dropPendingBitWhenBeatHasData(io.inner.release)) + val curr_read_beat = PriorityEncoder(pending_reads) + io.data.read.valid := state === s_busy && pending_reads.orR io.data.read.bits.id := UInt(trackerId) - io.data.read.bits.way_en := xact_way_en - io.data.read.bits.addr_idx := xact_addr_block(idxMSB,idxLSB) - io.data.read.bits.addr_beat := read_data_cnt + io.data.read.bits.way_en := xact.way_en + io.data.read.bits.addr_idx := xact.idx + io.data.read.bits.addr_beat := curr_read_beat io.data.write.valid := Bool(false) - io.wb.req.ready := Bool(false) - io.wb.resp.valid := Bool(false) - io.wb.resp.bits.id := xact_id - - switch (state) { - is(s_idle) { - io.wb.req.ready := Bool(true) - when(io.wb.req.valid) { - xact_addr_block := io.wb.req.bits.addr_block - xact_coh := io.wb.req.bits.coh - xact_way_en := io.wb.req.bits.way_en - xact_id := io.wb.req.bits.id - irel_had_data := Bool(false) - val needs_inner_probes = io.wb.req.bits.coh.inner.requiresProbesOnVoluntaryWriteback() - when(needs_inner_probes) { - pending_probes := mask_incoherent - irel_cnt := PopCount(mask_incoherent) - } - state := Mux(needs_inner_probes, s_inner_probe, s_data_read) - } - } - is(s_inner_probe) { - // Send probes - io.inner.probe.valid := pending_probes != UInt(0) - when(io.inner.probe.ready) { - pending_probes := pending_probes & ~UIntToOH(curr_probe_dst) - } - // Handle releases, which may have data being written back - io.inner.release.ready := Bool(true) - when(io.inner.release.valid) { - xact_coh.inner := pending_icoh_on_irel - // Handle released dirty data - when(io.irel().hasData()) { - irel_had_data := Bool(true) - xact_coh.outer := pending_ocoh_on_irel - data_buffer(io.irel().addr_beat) := io.irel().data - } - // We don't decrement irel_cnt until we've received all the data beats. - when(!io.irel().hasData() || irel_data_done) { - irel_cnt := irel_cnt - UInt(1) - } - } - when(irel_cnt === UInt(0)) { - state := Mux(irel_had_data, // If someone released a dirty block - s_outer_release, // write that block back, otherwise - Mux(xact_coh.outer.requiresVoluntaryWriteback(), - s_data_read, // write extant dirty data back, or just - s_wb_resp)) // drop a clean block after collecting acks - } - } - is(s_data_read) { - io.data.read.valid := Bool(true) - when(io.data.resp.valid) { data_buffer(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } - when(read_data_done) { state := s_data_resp } - } - is(s_data_resp) { - when(io.data.resp.valid) { data_buffer(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } - when(resp_data_done) { state := s_outer_release } - } - is(s_outer_release) { - io.outer.release.valid := Bool(true) - when(orel_data_done) { - state := Mux(io.orel().requiresAck(), s_outer_grant, s_wb_resp) - } - } - is(s_outer_grant) { - io.outer.grant.ready := Bool(true) - when(io.outer.grant.valid) { - state := s_wb_resp - } - } - is(s_wb_resp) { - io.wb.resp.valid := Bool(true) - state := s_idle - } + pending_resps := (pending_resps & dropPendingBitInternal(io.data.resp)) | + addPendingBitInternal(io.data.read) + when(io.data.resp.valid) { + data_buffer(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } + + // Once the data is buffered we can write it back to outer memory + pending_orel_data := pending_orel_data | + addPendingBitWhenBeatHasData(io.inner.release) | + addPendingBitInternal(io.data.resp) + io.outer.release.valid := state === s_busy && + (!io.orel().hasData() || pending_orel_data(orel_data_idx)) + io.outer.release.bits := xact.coh.outer.makeVoluntaryWriteback( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block, + addr_beat = orel_data_idx, + data = data_buffer(orel_data_idx)) + + // Wait for an acknowledgement + io.outer.grant.ready := state === s_outer_grant + + // Respond to the initiating transaction handler signalling completion of the writeback + io.wb.resp.valid := state === s_wb_resp + io.wb.resp.bits.id := xact.id + + // State machine updates and transaction handler metadata intialization + when(state === s_idle && io.wb.req.valid) { + xact := io.wb.req.bits + val coh = io.wb.req.bits.coh + val needs_inner_probes = coh.inner.requiresProbesOnVoluntaryWriteback() + when(needs_inner_probes) { pending_iprbs := coh.inner.full() & ~io.incoherent.toBits } + pending_reads := SInt(-1, width = innerDataBeats) + pending_resps := UInt(0) + pending_orel_data := UInt(0) + state := Mux(needs_inner_probes, s_inner_probe, s_busy) + } + when(state === s_inner_probe && !(pending_iprbs.orR || pending_irels)) { + state := Mux(xact.coh.outer.requiresVoluntaryWriteback(), s_busy, s_wb_resp) + } + when(state === s_busy && orel_data_done) { + state := Mux(io.orel().requiresAck(), s_outer_grant, s_wb_resp) + } + when(state === s_outer_grant && ognt_data_done) { state := s_wb_resp } + when(state === s_wb_resp ) { state := s_idle } + + // These IOs are used for routing in the parent + io.has_release_match := io.irel().conflicts(xact_addr_block) && !io.irel().isVoluntary() && io.inner.release.ready + io.has_acquire_match := Bool(false) + io.has_acquire_conflict := Bool(false) + + // IOs pinned low + io.outer.acquire.valid := Bool(false) + io.outer.probe.ready := Bool(false) // TODO: handle these here + io.inner.grant.valid := Bool(false) + io.inner.acquire.ready := Bool(false) + io.inner.finish.ready := Bool(false) } From 09e30041edd97d9e3e6dfa0bb779c64c4234a770 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 27 Apr 2015 12:56:33 -0700 Subject: [PATCH 372/374] Voluntary Writeback tracker rewrite --- uncore/src/main/scala/cache.scala | 212 ++++++++++++++---------------- 1 file changed, 97 insertions(+), 115 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 32529202..7fcd1408 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -337,8 +337,7 @@ class L2DataArray(delay: Int) extends L2HellaCacheModule { io.write.ready := Bool(true) } -class L2HellaCacheBank extends HierarchicalCoherenceAgent - with L2HellaCacheParameters { +class L2HellaCacheBank extends HierarchicalCoherenceAgent with L2HellaCacheParameters { require(isPow2(nSets)) require(isPow2(nWays)) @@ -357,8 +356,7 @@ class TSHRFileIO extends HierarchicalTLIO { val data = new L2DataRWIO } -class TSHRFile extends L2HellaCacheModule - with HasCoherenceAgentWiringHelpers { +class TSHRFile extends L2HellaCacheModule with HasCoherenceAgentWiringHelpers { val io = new TSHRFileIO // Create TSHRs for outstanding transactions @@ -381,7 +379,6 @@ class TSHRFile extends L2HellaCacheModule val acquire_idx = Mux(acquireMatches.orR, PriorityEncoder(acquireMatches), PriorityEncoder(acquireReadys)) - val block_acquires = acquireConflicts.orR io.inner.acquire.ready := acquireReadys.orR && !block_acquires trackerAcquireIOs.zipWithIndex.foreach { @@ -432,7 +429,7 @@ class L2XactTrackerIO extends HierarchicalXactTrackerIO { } abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { - class CacheBlockBuffer { + class CacheBlockBuffer { // TODO val buffer = Reg(Bits(width = params(CacheBlockBytes)*8)) def internal = Vec.fill(internalDataBeats){ Bits(width = rowBits) }.fromBits(buffer) @@ -477,127 +474,118 @@ abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { val isPartial = a.wmask() != Acquire.fullWriteMask addPendingBitWhenBeat(in.fire() && isPartial && Bool(ignoresWriteMask), a) } + + def pinAllReadyValidLow[T <: Data](b: Bundle) { + b.elements.foreach { + _._2 match { + case d: DecoupledIO[T] => + if(d.ready.dir == OUTPUT) d.ready := Bool(false) + else if(d.valid.dir == OUTPUT) d.valid := Bool(false) + case v: ValidIO[T] => if(v.valid.dir == OUTPUT) v.valid := Bool(false) + case b: Bundle => pinAllReadyValidLow(b) + case _ => + } + } + } } class L2VoluntaryReleaseTracker(trackerId: Int) extends L2XactTracker { val io = new L2XactTrackerIO + pinAllReadyValidLow(io) - val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_inner_grant :: s_inner_finish :: Nil = Enum(UInt(), 7) + val s_idle :: s_meta_read :: s_meta_resp :: s_busy :: s_meta_write :: Nil = Enum(UInt(), 5) val state = Reg(init=s_idle) val xact = Reg(Bundle(new ReleaseFromSrc, { case TLId => params(InnerTLId); case TLDataBits => 0 })) - val xact_tag_match = Reg{ Bool() } - val xact_old_meta = Reg{ new L2Metadata } + val data_buffer = Vec.fill(innerDataBeats){ Reg(init=UInt(0, width = innerDataBits)) } val xact_way_en = Reg{ Bits(width = nWays) } - val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) } + val xact_old_meta = Reg{ new L2Metadata } val coh = xact_old_meta.coh - val collect_irel_data = Reg(init=Bool(false)) - val irel_data_valid = Reg(init=Bits(0, width = innerDataBeats)) - val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val (write_data_cnt, write_data_done) = connectInternalDataBeatCounter(io.data.write) + val pending_irels = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_writes = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_ignt = Reg(init=Bool(false)) - io.has_acquire_conflict := Bool(false) - io.has_acquire_match := Bool(false) - io.has_release_match := io.irel().isVoluntary() + val all_pending_done = + !(pending_writes.orR || + pending_ignt) - io.outer.acquire.valid := Bool(false) - io.outer.probe.ready := Bool(false) - io.outer.release.valid := Bool(false) - io.outer.grant.ready := Bool(false) - io.inner.acquire.ready := Bool(false) - io.inner.probe.valid := Bool(false) - io.inner.release.ready := Bool(false) - io.inner.grant.valid := Bool(false) - io.inner.finish.ready := Bool(false) + // Accept a voluntary Release (and any further beats of data) + pending_irels := (pending_irels & dropPendingBitWhenBeatHasData(io.inner.release)) + io.inner.release.ready := state === s_idle || pending_irels.orR + when(io.inner.release.fire()) { data_buffer(io.irel().addr_beat) := io.irel().data } - io.inner.grant.bits := coh.inner.makeGrant(xact, UInt(trackerId)) - - io.data.read.valid := Bool(false) - io.data.write.valid := Bool(false) - io.data.write.bits.id := UInt(trackerId) - io.data.write.bits.way_en := xact_way_en - io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) - io.data.write.bits.addr_beat := write_data_cnt - io.data.write.bits.wmask := SInt(-1) - io.data.write.bits.data := data_buffer(write_data_cnt) - io.meta.read.valid := Bool(false) + // Begin a transaction by getting the current block metadata + io.meta.read.valid := state === s_meta_read io.meta.read.bits.id := UInt(trackerId) io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) io.meta.read.bits.tag := xact.addr_block >> UInt(idxBits) - io.meta.write.valid := Bool(false) + + // Write the voluntarily written back data to this cache + pending_writes := (pending_writes & dropPendingBit(io.data.write)) | + addPendingBitWhenBeatHasData(io.inner.release) + val curr_write_beat = PriorityEncoder(pending_writes) + io.data.write.valid := state === s_busy && pending_writes.orR + io.data.write.bits.id := UInt(trackerId) + io.data.write.bits.way_en := xact_way_en + io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) + io.data.write.bits.addr_beat := curr_write_beat + io.data.write.bits.wmask := SInt(-1) + io.data.write.bits.data := data_buffer(curr_write_beat) + + // Send an acknowledgement + io.inner.grant.valid := state === s_busy && pending_ignt && !pending_irels + io.inner.grant.bits := coh.inner.makeGrant(xact, UInt(trackerId)) + when(io.inner.grant.fire()) { pending_ignt := Bool(false) } + + // End a transaction by updating the block metadata + io.meta.write.valid := state === s_meta_write io.meta.write.bits.id := UInt(trackerId) io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en io.meta.write.bits.data.tag := xact.addr_block >> UInt(idxBits) io.meta.write.bits.data.coh.inner := xact_old_meta.coh.inner.onRelease(xact) - io.meta.write.bits.data.coh.outer := xact_old_meta.coh.outer.onHit(M_XWR) // WB is a write - io.wb.req.valid := Bool(false) + io.meta.write.bits.data.coh.outer := Mux(xact.hasData(), + xact_old_meta.coh.outer.onHit(M_XWR), + xact_old_meta.coh.outer) - when(collect_irel_data) { - io.inner.release.ready := Bool(true) - when(io.inner.release.valid) { - data_buffer(io.irel().addr_beat) := io.irel().data - irel_data_valid(io.irel().addr_beat) := Bool(true) + // State machine updates and transaction handler metadata intialization + when(state === s_idle && io.inner.release.valid) { + xact := io.irel() + when(io.irel().hasMultibeatData()) { + pending_irels := dropPendingBitWhenBeatHasData(io.inner.release) + }. otherwise { + pending_irels := UInt(0) } - when(irel_data_done) { collect_irel_data := Bool(false) } + pending_writes := addPendingBitWhenBeatHasData(io.inner.release) + pending_ignt := io.irel().requiresAck() + state := s_meta_read } + when(state === s_meta_read && io.meta.read.ready) { state := s_meta_resp } + when(state === s_meta_resp && io.meta.resp.valid) { + xact_old_meta := io.meta.resp.bits.meta + xact_way_en := io.meta.resp.bits.way_en + state := s_busy + } + when(state === s_busy && all_pending_done) { state := s_meta_write } + when(state === s_meta_write && io.meta.write.ready) { state := s_idle } - switch (state) { - is(s_idle) { - io.inner.release.ready := Bool(true) - when( io.inner.release.valid ) { - xact := io.irel() - data_buffer(io.irel().addr_beat) := io.irel().data - collect_irel_data := io.irel().hasMultibeatData() - irel_data_valid := io.irel().hasData() << io.irel().addr_beat - state := s_meta_read - } - } - is(s_meta_read) { - io.meta.read.valid := Bool(true) - when(io.meta.read.ready) { state := s_meta_resp } - } - is(s_meta_resp) { - when(io.meta.resp.valid) { - xact_tag_match := io.meta.resp.bits.tag_match - xact_old_meta := io.meta.resp.bits.meta - xact_way_en := io.meta.resp.bits.way_en - state := Mux(io.meta.resp.bits.tag_match, - Mux(xact.hasData(), s_data_write, s_meta_write), - Mux(xact.requiresAck(), s_inner_grant, s_idle)) - } - } - is(s_data_write) { - io.data.write.valid := !collect_irel_data || irel_data_valid(write_data_cnt) - when(write_data_done) { state := s_meta_write } - } - is(s_meta_write) { - io.meta.write.valid := Bool(true) - when(io.meta.write.ready) { - state := Mux(xact.requiresAck(), s_inner_grant, s_idle) // Need a Grant.voluntaryAck? - } - } - is(s_inner_grant) { - io.inner.grant.valid := !collect_irel_data - when(io.inner.grant.ready) { - state := Mux(io.ignt().requiresAck(), s_inner_finish, s_idle) - } - } - is(s_inner_finish) { - io.inner.finish.ready := Bool(true) - when(io.inner.finish.valid) { state := s_idle } - } - } + // These IOs are used for routing in the parent + io.has_release_match := io.irel().isVoluntary() + io.has_acquire_match := Bool(false) + io.has_acquire_conflict := Bool(false) // Checks for illegal behavior assert(!(state === s_meta_resp && io.meta.resp.valid && !io.meta.resp.bits.tag_match), "VoluntaryReleaseTracker accepted Release for a block not resident in this cache!") + assert(!(state === s_idle && io.inner.release.fire() && !io.irel().isVoluntary()), + "VoluntaryReleaseTracker accepted Release that wasn't voluntary!") } class L2AcquireTracker(trackerId: Int) extends L2XactTracker { val io = new L2XactTrackerIO + pinAllReadyValidLow(io) val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_inner_probe :: s_outer_acquire :: s_busy :: s_meta_write :: Nil = Enum(UInt(), 9) val state = Reg(init=s_idle) @@ -742,6 +730,20 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { // Track whether any beats are missing from a PutBlock pending_puts := (pending_puts & dropPendingBitWhenBeatHasData(io.inner.acquire)) + // Begin a transaction by getting the current block metadata + io.meta.read.valid := state === s_meta_read + io.meta.read.bits.id := UInt(trackerId) + io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) + io.meta.read.bits.tag := xact.addr_block >> UInt(idxBits) + + // Issue a request to the writeback unit + io.wb.req.valid := state === s_wb_req + io.wb.req.bits.id := UInt(trackerId) + io.wb.req.bits.idx := xact.addr_block(idxMSB,idxLSB) + io.wb.req.bits.tag := xact_old_meta.tag + io.wb.req.bits.coh := xact_old_meta.coh + io.wb.req.bits.way_en := xact_way_en + // Track which clients yet need to be probed and make Probe message pending_iprbs := pending_iprbs & dropPendingBitAtDest(io.inner.probe) val curr_probe_dst = PriorityEncoder(pending_iprbs) @@ -773,10 +775,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { op_code = xact.op_code()), Bundle(Acquire(xact))(outerTLParams)) - // Probes from the outer memory system are handled in the WritebackUnit - io.outer.probe.ready := Bool(false) - io.outer.release.valid := Bool(false) - + // Handle the response from outer memory io.outer.grant.ready := state === s_busy val pending_coh_on_ognt = HierarchicalMetadata( ManagerMetadata.onReset, @@ -853,11 +852,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { io.data.write.bits.wmask := wmask_buffer(curr_write_beat) io.data.write.bits.data := data_buffer(curr_write_beat) - io.meta.read.valid := state === s_meta_read - io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) - io.meta.read.bits.tag := xact.addr_block >> UInt(idxBits) - + // End a transaction by updating the block metadata io.meta.write.valid := state === s_meta_write io.meta.write.bits.id := UInt(trackerId) io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) @@ -865,13 +860,6 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { io.meta.write.bits.data.tag := xact.addr_block >> UInt(idxBits) io.meta.write.bits.data.coh := pending_coh - io.wb.req.valid := state === s_wb_req - io.wb.req.bits.id := UInt(trackerId) - io.wb.req.bits.idx := xact.addr_block(idxMSB,idxLSB) - io.wb.req.bits.tag := xact_old_meta.tag - io.wb.req.bits.coh := xact_old_meta.coh - io.wb.req.bits.way_en := xact_way_en - // Handling of secondary misses (Gets and Puts only for now) when(io.inner.acquire.fire() && io.iacq().hasData()) { // state <= s_meta_wrtie val beat = io.iacq().addr_beat @@ -988,6 +976,7 @@ class L2WritebackUnitIO extends HierarchicalXactTrackerIO { class L2WritebackUnit(trackerId: Int) extends L2XactTracker { val io = new L2WritebackUnitIO + pinAllReadyValidLow(io) val s_idle :: s_inner_probe :: s_busy :: s_outer_grant :: s_wb_resp :: Nil = Enum(UInt(), 5) val state = Reg(init=s_idle) @@ -1086,11 +1075,4 @@ class L2WritebackUnit(trackerId: Int) extends L2XactTracker { io.has_release_match := io.irel().conflicts(xact_addr_block) && !io.irel().isVoluntary() && io.inner.release.ready io.has_acquire_match := Bool(false) io.has_acquire_conflict := Bool(false) - - // IOs pinned low - io.outer.acquire.valid := Bool(false) - io.outer.probe.ready := Bool(false) // TODO: handle these here - io.inner.grant.valid := Bool(false) - io.inner.acquire.ready := Bool(false) - io.inner.finish.ready := Bool(false) } From 3673295d03672b2a1fcc96e7b402de571c7bf7ee Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 27 Apr 2015 16:59:30 -0700 Subject: [PATCH 373/374] network shim cleanup --- uncore/src/main/scala/network.scala | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index f61cec6c..4b00091d 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -57,7 +57,10 @@ class LogicalNetworkIO[T <: Data](dType: T) extends Bundle { } object DecoupledLogicalNetworkIOWrapper { - def apply[T <: Data](in: DecoupledIO[T], src: UInt = UInt(0), dst: UInt = UInt(0)) = { + def apply[T <: Data]( + in: DecoupledIO[T], + src: UInt = UInt(0), + dst: UInt = UInt(0)): DecoupledIO[LogicalNetworkIO[T]] = { val out = Decoupled(new LogicalNetworkIO(in.bits.clone)).asDirectionless out.valid := in.valid out.bits.payload := in.bits @@ -69,7 +72,7 @@ object DecoupledLogicalNetworkIOWrapper { } object DecoupledLogicalNetworkIOUnwrapper { - def apply[T <: Data](in: DecoupledIO[LogicalNetworkIO[T]]) = { + def apply[T <: Data](in: DecoupledIO[LogicalNetworkIO[T]]): DecoupledIO[T] = { val out = Decoupled(in.bits.payload.clone).asDirectionless out.valid := in.valid out.bits := in.bits.payload @@ -77,3 +80,25 @@ object DecoupledLogicalNetworkIOUnwrapper { out } } + +object DefaultFromPhysicalShim { + def apply[T <: Data](in: DecoupledIO[PhysicalNetworkIO[T]]): DecoupledIO[LogicalNetworkIO[T]] = { + val out = Decoupled(new LogicalNetworkIO(in.bits.payload)).asDirectionless + out.bits.header := in.bits.header + out.bits.payload := in.bits.payload + out.valid := in.valid + in.ready := out.ready + out + } +} + +object DefaultToPhysicalShim { + def apply[T <: Data](n: Int, in: DecoupledIO[LogicalNetworkIO[T]]): DecoupledIO[PhysicalNetworkIO[T]] = { + val out = Decoupled(new PhysicalNetworkIO(n, in.bits.payload)).asDirectionless + out.bits.header := in.bits.header + out.bits.payload := in.bits.payload + out.valid := in.valid + in.ready := out.ready + out + } +} From 4cef8c9cd4b1334d9705932f3ba226960b0dabd3 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 7 May 2015 10:55:38 -0700 Subject: [PATCH 374/374] Added MemIOArbiter --- uncore/src/main/scala/memserdes.scala | 65 +++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 7497e2e4..a4944e1e 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -60,14 +60,14 @@ class MemPipeIO extends Bundle { val resp = Valid(new MemResp).flip } -class MemSerializedIO(w: Int) extends Bundle -{ +class MemSerializedIO(w: Int) extends Bundle { val req = Decoupled(Bits(width = w)) val resp = Valid(Bits(width = w)).flip } -class MemSerdes(w: Int) extends MIFModule +class MemSerdes extends MIFModule { + val w = params(HTIFWidth) val io = new Bundle { val wide = new MemIO().flip val narrow = new MemSerializedIO(w) @@ -517,6 +517,65 @@ object HellaQueue } } +class MemIOArbiter(val arbN: Int) extends MIFModule { + val io = new Bundle { + val inner = Vec.fill(arbN){new MemIO}.flip + val outer = new MemIO + } + + if(arbN > 1) { + val cmd_arb = Module(new RRArbiter(new MemReqCmd, arbN)) + val choice_q = Module(new Queue(cmd_arb.io.chosen.clone, 4)) + val (data_cnt, data_done) = Counter(io.outer.req_data.fire(), mifDataBeats) + + io.inner.map(_.req_cmd).zipWithIndex.zip(cmd_arb.io.in).map{ case ((req, id), arb) => { + arb.valid := req.valid + arb.bits := req.bits + arb.bits.tag := Cat(req.bits.tag, UInt(id)) + req.ready := arb.ready + }} + io.outer.req_cmd.bits := cmd_arb.io.out.bits + io.outer.req_cmd.valid := cmd_arb.io.out.valid && choice_q.io.enq.ready + cmd_arb.io.out.ready := io.outer.req_cmd.ready && choice_q.io.enq.ready + choice_q.io.enq.bits := cmd_arb.io.chosen + choice_q.io.enq.valid := cmd_arb.io.out.fire() && cmd_arb.io.out.bits.rw + + io.outer.req_data.bits := io.inner(choice_q.io.deq.bits).req_data.bits + io.outer.req_data.valid := io.inner(choice_q.io.deq.bits).req_data.valid && choice_q.io.deq.valid + io.inner.map(_.req_data.ready).zipWithIndex.foreach { + case(r, i) => r := UInt(i) === choice_q.io.deq.bits && choice_q.io.deq.valid + } + choice_q.io.deq.ready := data_done + + io.outer.resp.ready := Bool(false) + for (i <- 0 until arbN) { + io.inner(i).resp.valid := Bool(false) + when(io.outer.resp.bits.tag(log2Up(arbN)-1,0).toUInt === UInt(i)) { + io.inner(i).resp.valid := io.outer.resp.valid + io.outer.resp.ready := io.inner(i).resp.ready + } + io.inner(i).resp.bits := io.outer.resp.bits + io.inner(i).resp.bits.tag := io.outer.resp.bits.tag >> UInt(log2Up(arbN)) + } + } else { io.inner.head <> io.outer } +} + +object MemIOMemPipeIOConverter { + def apply(in: MemPipeIO): MemIO = { + val out = new MemIO().asDirectionless + in.resp.valid := out.resp.valid + in.resp.bits := out.resp.bits + out.resp.ready := Bool(true) + out.req_cmd.valid := in.req_cmd.valid + out.req_cmd.bits := in.req_cmd.bits + in.req_cmd.ready := out.req_cmd.ready + out.req_data.valid := in.req_data.valid + out.req_data.bits := in.req_data.bits + in.req_data.ready := out.req_data.ready + out + } +} + class MemPipeIOMemIOConverter(numRequests: Int) extends MIFModule { val io = new Bundle { val cpu = new MemIO().flip